diff --git a/BUILD.bazel b/BUILD.bazel index cc623aac7..98584b604 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,384 +1,22 @@ # Bazel (https://bazel.build/) BUILD file for Protobuf. load("@bazel_skylib//rules:common_settings.bzl", "string_flag") -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test", "objc_library", native_cc_proto_library = "cc_proto_library") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "cc_test") load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain", "proto_library") -load("@rules_python//python:defs.bzl", "py_library") load("@rules_java//java:defs.bzl", "java_binary", "java_lite_proto_library", "java_proto_library") -load(":cc_proto_blacklist_test.bzl", "cc_proto_blacklist_test") -load(":protobuf_release.bzl", "package_naming") +load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") +load(":protobuf.bzl", "py_proto_library") licenses(["notice"]) exports_files(["LICENSE"]) -################################################################################ -# build configuration -################################################################################ - -################################################################################ -# ZLIB configuration -################################################################################ - -ZLIB_DEPS = ["@zlib//:zlib"] - -################################################################################ -# Protobuf Runtime Library -################################################################################ - -MSVC_COPTS = [ - "/wd4065", # switch statement contains 'default' but no 'case' labels - "/wd4244", # 'conversion' conversion from 'type1' to 'type2', possible loss of data - "/wd4251", # 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2' - "/wd4267", # 'var' : conversion from 'size_t' to 'type', possible loss of data - "/wd4305", # 'identifier' : truncation from 'type1' to 'type2' - "/wd4307", # 'operator' : integral constant overflow - "/wd4309", # 'conversion' : truncation of constant value - "/wd4334", # 'operator' : result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?) - "/wd4355", # 'this' : used in base member initializer list - "/wd4506", # no definition for inline function 'function' - "/wd4800", # 'type' : forcing value to bool 'true' or 'false' (performance warning) - "/wd4996", # The compiler encountered a deprecated declaration. -] - -COPTS = select({ - ":msvc": MSVC_COPTS, - "//conditions:default": [ - "-DHAVE_ZLIB", - "-Woverloaded-virtual", - "-Wno-sign-compare", - ], -}) - -load(":compiler_config_setting.bzl", "create_compiler_config_setting") - -create_compiler_config_setting( - name = "msvc", - value = "msvc-cl", - visibility = [ - # Public, but Protobuf only visibility. - "//:__subpackages__", - ], -) - -# Android NDK builds can specify different crosstool_top flags to choose which -# STL they use for C++. We need these multiple variants to catch all of those -# versions of crosstool_top and reliably detect Android. -# -# For more info on the various crosstool_tops used by NDK Bazel builds, see: -# https://docs.bazel.build/versions/master/android-ndk.html#configuring-the-stl - -config_setting( - name = "android", - values = { - "crosstool_top": "//external:android/crosstool", - }, - visibility = [ - # Public, but Protobuf only visibility. - "//:__subpackages__", - ], -) - -config_setting( - name = "android-stlport", - values = { - "crosstool_top": "@androidndk//:toolchain-stlport", - }, - visibility = [ - # Public, but Protobuf only visibility. - "//:__subpackages__", - ], -) - -config_setting( - name = "android-libcpp", - values = { - "crosstool_top": "@androidndk//:toolchain-libcpp", - }, - visibility = [ - # Public, but Protobuf only visibility. - "//:__subpackages__", - ], -) - -config_setting( - name = "android-gnu-libstdcpp", - values = { - "crosstool_top": "@androidndk//:toolchain-gnu-libstdcpp", - }, - visibility = [ - # Public, but Protobuf only visibility. - "//:__subpackages__", - ], -) - -config_setting( - name = "android-default", - values = { - "crosstool_top": "@androidndk//:default_crosstool", - }, - visibility = [ - # Public, but Protobuf only visibility. - "//:__subpackages__", - ], -) - -# Android and MSVC builds do not need to link in a separate pthread library. -LINK_OPTS = select({ - ":android": [], - ":android-stlport": [], - ":android-libcpp": [], - ":android-gnu-libstdcpp": [], - ":android-default": [], - ":msvc": [ - # Suppress linker warnings about files with no symbols defined. - "-ignore:4221", - ], - "//conditions:default": [ - "-lpthread", - "-lm", - ], -}) - -load( - ":protobuf.bzl", - "adapt_proto_library", - "cc_proto_library", - "internal_copied_filegroup", - "internal_gen_kt_protos", - "internal_gen_well_known_protos_java", - "internal_protobuf_py_tests", - "py_proto_library", -) - -cc_library( - name = "protobuf_lite", - srcs = [ - # AUTOGEN(protobuf_lite_srcs) - "src/google/protobuf/any_lite.cc", - "src/google/protobuf/arena.cc", - "src/google/protobuf/arenastring.cc", - "src/google/protobuf/arenaz_sampler.cc", - "src/google/protobuf/extension_set.cc", - "src/google/protobuf/generated_enum_util.cc", - "src/google/protobuf/generated_message_tctable_lite.cc", - "src/google/protobuf/generated_message_util.cc", - "src/google/protobuf/implicit_weak_message.cc", - "src/google/protobuf/inlined_string_field.cc", - "src/google/protobuf/io/coded_stream.cc", - "src/google/protobuf/io/io_win32.cc", - "src/google/protobuf/io/strtod.cc", - "src/google/protobuf/io/zero_copy_stream.cc", - "src/google/protobuf/io/zero_copy_stream_impl.cc", - "src/google/protobuf/io/zero_copy_stream_impl_lite.cc", - "src/google/protobuf/map.cc", - "src/google/protobuf/message_lite.cc", - "src/google/protobuf/parse_context.cc", - "src/google/protobuf/repeated_field.cc", - "src/google/protobuf/repeated_ptr_field.cc", - "src/google/protobuf/stubs/bytestream.cc", - "src/google/protobuf/stubs/common.cc", - "src/google/protobuf/stubs/int128.cc", - "src/google/protobuf/stubs/status.cc", - "src/google/protobuf/stubs/statusor.cc", - "src/google/protobuf/stubs/stringpiece.cc", - "src/google/protobuf/stubs/stringprintf.cc", - "src/google/protobuf/stubs/structurally_valid.cc", - "src/google/protobuf/stubs/strutil.cc", - "src/google/protobuf/stubs/time.cc", - "src/google/protobuf/wire_format_lite.cc", - ], - hdrs = glob([ - "src/google/protobuf/**/*.h", - "src/google/protobuf/**/*.inc", - ]), - copts = COPTS, - includes = ["src/"], - linkopts = LINK_OPTS, - visibility = ["//visibility:public"], -) - -PROTOBUF_DEPS = select({ - ":msvc": [], - "//conditions:default": ZLIB_DEPS, -}) - -cc_library( - name = "protobuf", - srcs = [ - # AUTOGEN(protobuf_srcs) - "src/google/protobuf/any.cc", - "src/google/protobuf/any.pb.cc", - "src/google/protobuf/api.pb.cc", - "src/google/protobuf/compiler/importer.cc", - "src/google/protobuf/compiler/parser.cc", - "src/google/protobuf/descriptor.cc", - "src/google/protobuf/descriptor.pb.cc", - "src/google/protobuf/descriptor_database.cc", - "src/google/protobuf/duration.pb.cc", - "src/google/protobuf/dynamic_message.cc", - "src/google/protobuf/empty.pb.cc", - "src/google/protobuf/extension_set_heavy.cc", - "src/google/protobuf/field_mask.pb.cc", - "src/google/protobuf/generated_message_bases.cc", - "src/google/protobuf/generated_message_reflection.cc", - "src/google/protobuf/generated_message_tctable_full.cc", - "src/google/protobuf/io/gzip_stream.cc", - "src/google/protobuf/io/printer.cc", - "src/google/protobuf/io/tokenizer.cc", - "src/google/protobuf/map_field.cc", - "src/google/protobuf/message.cc", - "src/google/protobuf/reflection_ops.cc", - "src/google/protobuf/service.cc", - "src/google/protobuf/source_context.pb.cc", - "src/google/protobuf/struct.pb.cc", - "src/google/protobuf/stubs/substitute.cc", - "src/google/protobuf/text_format.cc", - "src/google/protobuf/timestamp.pb.cc", - "src/google/protobuf/type.pb.cc", - "src/google/protobuf/unknown_field_set.cc", - "src/google/protobuf/util/delimited_message_util.cc", - "src/google/protobuf/util/field_comparator.cc", - "src/google/protobuf/util/field_mask_util.cc", - "src/google/protobuf/util/internal/datapiece.cc", - "src/google/protobuf/util/internal/default_value_objectwriter.cc", - "src/google/protobuf/util/internal/error_listener.cc", - "src/google/protobuf/util/internal/field_mask_utility.cc", - "src/google/protobuf/util/internal/json_escaping.cc", - "src/google/protobuf/util/internal/json_objectwriter.cc", - "src/google/protobuf/util/internal/json_stream_parser.cc", - "src/google/protobuf/util/internal/object_writer.cc", - "src/google/protobuf/util/internal/proto_writer.cc", - "src/google/protobuf/util/internal/protostream_objectsource.cc", - "src/google/protobuf/util/internal/protostream_objectwriter.cc", - "src/google/protobuf/util/internal/type_info.cc", - "src/google/protobuf/util/internal/utility.cc", - "src/google/protobuf/util/json_util.cc", - "src/google/protobuf/util/message_differencer.cc", - "src/google/protobuf/util/time_util.cc", - "src/google/protobuf/util/type_resolver_util.cc", - "src/google/protobuf/wire_format.cc", - "src/google/protobuf/wrappers.pb.cc", - ], - hdrs = glob([ - "src/**/*.h", - "src/**/*.inc", - ]), - copts = COPTS, - includes = ["src/"], - linkopts = LINK_OPTS, - visibility = ["//visibility:public"], - deps = [":protobuf_lite"] + PROTOBUF_DEPS, -) - -# This provides just the header files for use in projects that need to build -# shared libraries for dynamic loading. This target is available until Bazel -# adds native support for such use cases. -# TODO(keveman): Remove this target once the support gets added to Bazel. -cc_library( - name = "protobuf_headers", - hdrs = glob([ - "src/**/*.h", - "src/**/*.inc", - ]), - includes = ["src/"], - visibility = ["//visibility:public"], -) - -# Map of all well known protos. -# name => (include path, imports) -WELL_KNOWN_PROTO_MAP = { - "any": ("src/google/protobuf/any.proto", []), - "api": ( - "src/google/protobuf/api.proto", - [ - "source_context", - "type", - ], - ), - "compiler_plugin": ( - "src/google/protobuf/compiler/plugin.proto", - ["descriptor"], - ), - "descriptor": ("src/google/protobuf/descriptor.proto", []), - "duration": ("src/google/protobuf/duration.proto", []), - "empty": ("src/google/protobuf/empty.proto", []), - "field_mask": ("src/google/protobuf/field_mask.proto", []), - "source_context": ("src/google/protobuf/source_context.proto", []), - "struct": ("src/google/protobuf/struct.proto", []), - "timestamp": ("src/google/protobuf/timestamp.proto", []), - "type": ( - "src/google/protobuf/type.proto", - [ - "any", - "source_context", - ], - ), - "wrappers": ("src/google/protobuf/wrappers.proto", []), -} - -WELL_KNOWN_PROTOS = [value[0] for value in WELL_KNOWN_PROTO_MAP.values()] - -LITE_WELL_KNOWN_PROTO_MAP = { - "any": ("src/google/protobuf/any.proto", []), - "api": ( - "src/google/protobuf/api.proto", - [ - "source_context", - "type", - ], - ), - "duration": ("src/google/protobuf/duration.proto", []), - "empty": ("src/google/protobuf/empty.proto", []), - "field_mask": ("src/google/protobuf/field_mask.proto", []), - "source_context": ("src/google/protobuf/source_context.proto", []), - "struct": ("src/google/protobuf/struct.proto", []), - "timestamp": ("src/google/protobuf/timestamp.proto", []), - "type": ( - "src/google/protobuf/type.proto", - [ - "any", - "source_context", - ], - ), - "wrappers": ("src/google/protobuf/wrappers.proto", []), -} - -LITE_WELL_KNOWN_PROTOS = [value[0] for value in LITE_WELL_KNOWN_PROTO_MAP.values()] - -filegroup( - name = "well_known_protos", - srcs = WELL_KNOWN_PROTOS, - visibility = ["//visibility:public"], -) - -exports_files( - srcs = WELL_KNOWN_PROTOS, - visibility = ["//pkg:__pkg__"], -) - -filegroup( - name = "lite_well_known_protos", - srcs = LITE_WELL_KNOWN_PROTOS, - visibility = ["//visibility:public"], -) - -adapt_proto_library( - name = "cc_wkt_protos_genproto", - visibility = ["//visibility:public"], - deps = [proto + "_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()], -) - -cc_library( - name = "cc_wkt_protos", - deprecation = "Only for backward compatibility. Do not use.", - visibility = ["//visibility:public"], -) - ################################################################################ # Well Known Types Proto Library Rules # +# https://developers.google.com/protocol-buffers/docs/reference/google.protobuf +################################################################################ # These proto_library rules can be used with one of the language specific proto # library rules i.e. java_proto_library: # @@ -388,477 +26,174 @@ cc_library( # ) ################################################################################ -[proto_library( - name = proto[0] + "_proto", - srcs = [proto[1][0]], - strip_import_prefix = "src", +alias( + name = "any_proto", + actual = "//src/google/protobuf:any_proto", visibility = ["//visibility:public"], - deps = [dep + "_proto" for dep in proto[1][1]], -) for proto in WELL_KNOWN_PROTO_MAP.items()] +) -[native_cc_proto_library( - name = proto + "_cc_proto", - visibility = ["//visibility:private"], - deps = [proto + "_proto"], -) for proto in WELL_KNOWN_PROTO_MAP.keys()] +alias( + name = "api_proto", + actual = "//src/google/protobuf:api_proto", + visibility = ["//visibility:public"], +) -cc_proto_blacklist_test( - name = "cc_proto_blacklist_test", - tags = [ - # Exclude this target from wildcard expansion (//...). Due to - # https://github.com/bazelbuild/bazel/issues/10590, this test has to - # be nominated using the `@com_google_protobuf//` prefix. We do that, - # e.g., in kokoro/linux/bazel/build.sh. - # See also https://github.com/protocolbuffers/protobuf/pull/7096. - "manual", - ], - deps = [proto + "_cc_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()], +alias( + name = "duration_proto", + actual = "//src/google/protobuf:duration_proto", + visibility = ["//visibility:public"], +) + +alias( + name = "empty_proto", + actual = "//src/google/protobuf:empty_proto", + visibility = ["//visibility:public"], +) + +alias( + name = "field_mask_proto", + actual = "//src/google/protobuf:field_mask_proto", + visibility = ["//visibility:public"], +) + +alias( + name = "source_context_proto", + actual = "//src/google/protobuf:source_context_proto", + visibility = ["//visibility:public"], +) + +alias( + name = "struct_proto", + actual = "//src/google/protobuf:struct_proto", + visibility = ["//visibility:public"], +) + +alias( + name = "timestamp_proto", + actual = "//src/google/protobuf:timestamp_proto", + visibility = ["//visibility:public"], +) + +alias( + name = "type_proto", + actual = "//src/google/protobuf:type_proto", + visibility = ["//visibility:public"], +) + +alias( + name = "wrappers_proto", + actual = "//src/google/protobuf:wrappers_proto", + visibility = ["//visibility:public"], +) + +# Source files: these are aliases to a filegroup, not a `proto_library`. +# +# (This is _probably_ not what you want.) +alias( + name = "lite_well_known_protos", + actual = "//src/google/protobuf:well_known_type_protos", # filegroup + visibility = ["//visibility:public"], +) + +alias( + name = "well_known_type_protos", + actual = "//src/google/protobuf:well_known_type_protos", # filegroup + visibility = ["//visibility:public"], +) + +# Built-in runtime protos: these are part of protobuf's internal +# implementation, but are not Well-Known Types. + +alias( + name = "descriptor_proto", + actual = "//src/google/protobuf:descriptor_proto", # proto_library + visibility = ["//visibility:public"], +) + +alias( + name = "descriptor_proto_srcs", + actual = "//src/google/protobuf:descriptor_proto_srcs", # filegroup + visibility = ["//visibility:public"], +) + +alias( + name = "compiler_plugin_proto", + actual = "//src/google/protobuf/compiler:plugin_proto", # proto_library + visibility = ["//visibility:public"], +) + +cc_library( + name = "cc_wkt_protos", + deprecation = "Only for backward compatibility. Do not use.", + visibility = ["//visibility:public"], ) ################################################################################ # Protocol Buffers Compiler ################################################################################ -cc_library( - name = "protoc_lib", - srcs = [ - # AUTOGEN(protoc_lib_srcs) - "src/google/protobuf/compiler/code_generator.cc", - "src/google/protobuf/compiler/command_line_interface.cc", - "src/google/protobuf/compiler/cpp/enum.cc", - "src/google/protobuf/compiler/cpp/enum_field.cc", - "src/google/protobuf/compiler/cpp/extension.cc", - "src/google/protobuf/compiler/cpp/field.cc", - "src/google/protobuf/compiler/cpp/file.cc", - "src/google/protobuf/compiler/cpp/generator.cc", - "src/google/protobuf/compiler/cpp/helpers.cc", - "src/google/protobuf/compiler/cpp/map_field.cc", - "src/google/protobuf/compiler/cpp/message.cc", - "src/google/protobuf/compiler/cpp/message_field.cc", - "src/google/protobuf/compiler/cpp/padding_optimizer.cc", - "src/google/protobuf/compiler/cpp/parse_function_generator.cc", - "src/google/protobuf/compiler/cpp/primitive_field.cc", - "src/google/protobuf/compiler/cpp/service.cc", - "src/google/protobuf/compiler/cpp/string_field.cc", - "src/google/protobuf/compiler/csharp/csharp_doc_comment.cc", - "src/google/protobuf/compiler/csharp/csharp_enum.cc", - "src/google/protobuf/compiler/csharp/csharp_enum_field.cc", - "src/google/protobuf/compiler/csharp/csharp_field_base.cc", - "src/google/protobuf/compiler/csharp/csharp_generator.cc", - "src/google/protobuf/compiler/csharp/csharp_helpers.cc", - "src/google/protobuf/compiler/csharp/csharp_map_field.cc", - "src/google/protobuf/compiler/csharp/csharp_message.cc", - "src/google/protobuf/compiler/csharp/csharp_message_field.cc", - "src/google/protobuf/compiler/csharp/csharp_primitive_field.cc", - "src/google/protobuf/compiler/csharp/csharp_reflection_class.cc", - "src/google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc", - "src/google/protobuf/compiler/csharp/csharp_repeated_message_field.cc", - "src/google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc", - "src/google/protobuf/compiler/csharp/csharp_source_generator_base.cc", - "src/google/protobuf/compiler/csharp/csharp_wrapper_field.cc", - "src/google/protobuf/compiler/java/context.cc", - "src/google/protobuf/compiler/java/doc_comment.cc", - "src/google/protobuf/compiler/java/enum.cc", - "src/google/protobuf/compiler/java/enum_field.cc", - "src/google/protobuf/compiler/java/enum_field_lite.cc", - "src/google/protobuf/compiler/java/enum_lite.cc", - "src/google/protobuf/compiler/java/extension.cc", - "src/google/protobuf/compiler/java/extension_lite.cc", - "src/google/protobuf/compiler/java/field.cc", - "src/google/protobuf/compiler/java/file.cc", - "src/google/protobuf/compiler/java/generator.cc", - "src/google/protobuf/compiler/java/generator_factory.cc", - "src/google/protobuf/compiler/java/helpers.cc", - "src/google/protobuf/compiler/java/kotlin_generator.cc", - "src/google/protobuf/compiler/java/map_field.cc", - "src/google/protobuf/compiler/java/map_field_lite.cc", - "src/google/protobuf/compiler/java/message.cc", - "src/google/protobuf/compiler/java/message_builder.cc", - "src/google/protobuf/compiler/java/message_builder_lite.cc", - "src/google/protobuf/compiler/java/message_field.cc", - "src/google/protobuf/compiler/java/message_field_lite.cc", - "src/google/protobuf/compiler/java/message_lite.cc", - "src/google/protobuf/compiler/java/name_resolver.cc", - "src/google/protobuf/compiler/java/primitive_field.cc", - "src/google/protobuf/compiler/java/primitive_field_lite.cc", - "src/google/protobuf/compiler/java/service.cc", - "src/google/protobuf/compiler/java/shared_code_generator.cc", - "src/google/protobuf/compiler/java/string_field.cc", - "src/google/protobuf/compiler/java/string_field_lite.cc", - "src/google/protobuf/compiler/objectivec/objectivec_enum.cc", - "src/google/protobuf/compiler/objectivec/objectivec_enum_field.cc", - "src/google/protobuf/compiler/objectivec/objectivec_extension.cc", - "src/google/protobuf/compiler/objectivec/objectivec_field.cc", - "src/google/protobuf/compiler/objectivec/objectivec_file.cc", - "src/google/protobuf/compiler/objectivec/objectivec_generator.cc", - "src/google/protobuf/compiler/objectivec/objectivec_helpers.cc", - "src/google/protobuf/compiler/objectivec/objectivec_map_field.cc", - "src/google/protobuf/compiler/objectivec/objectivec_message.cc", - "src/google/protobuf/compiler/objectivec/objectivec_message_field.cc", - "src/google/protobuf/compiler/objectivec/objectivec_oneof.cc", - "src/google/protobuf/compiler/objectivec/objectivec_primitive_field.cc", - "src/google/protobuf/compiler/php/php_generator.cc", - "src/google/protobuf/compiler/plugin.cc", - "src/google/protobuf/compiler/plugin.pb.cc", - "src/google/protobuf/compiler/python/generator.cc", - "src/google/protobuf/compiler/python/helpers.cc", - "src/google/protobuf/compiler/python/pyi_generator.cc", - "src/google/protobuf/compiler/ruby/ruby_generator.cc", - "src/google/protobuf/compiler/subprocess.cc", - "src/google/protobuf/compiler/zip_writer.cc", - ], - copts = COPTS, - includes = ["src/"], - linkopts = LINK_OPTS, - visibility = ["//visibility:public"], - deps = [":protobuf"], -) - cc_binary( name = "protoc", - srcs = ["src/google/protobuf/compiler/main.cc"], linkopts = LINK_OPTS, visibility = ["//visibility:public"], - deps = [":protoc_lib"], + deps = ["//src/google/protobuf/compiler:protoc_lib"], ) ################################################################################ -# Tests +# C++ runtime ################################################################################ -filegroup( - name = "testdata", - srcs = glob(["src/google/protobuf/testdata/**/*"]), - visibility = [ - "//:__subpackages__", - "@upb//:__subpackages__", - ], +# The "lite" runtime works for .proto files that specify the option: +# optimize_for = LITE_RUNTIME; +# +# The lite runtime does not include the `Reflection` APIs (including +# `Descriptor` and related types) or Well-Known Types. +# +# See also: +# https://developers.google.com/protocol-buffers/docs/reference/cpp-generated#message +# https://developers.google.com/protocol-buffers/docs/reference/google.protobuf +alias( + name = "protobuf_lite", + actual = "//src/google/protobuf:protobuf_lite", + visibility = ["//visibility:public"], ) -RELATIVE_LITE_TEST_PROTOS = [ - # AUTOGEN(lite_test_protos) - "google/protobuf/map_lite_unittest.proto", - "google/protobuf/unittest_import_lite.proto", - "google/protobuf/unittest_import_public_lite.proto", - "google/protobuf/unittest_lite.proto", -] - -LITE_TEST_PROTOS = ["src/" + s for s in RELATIVE_LITE_TEST_PROTOS] - -RELATIVE_TEST_PROTOS = [ - # AUTOGEN(test_protos) - "google/protobuf/any_test.proto", - "google/protobuf/compiler/cpp/test_bad_identifiers.proto", - "google/protobuf/compiler/cpp/test_large_enum_value.proto", - "google/protobuf/map_proto2_unittest.proto", - "google/protobuf/map_unittest.proto", - "google/protobuf/unittest.proto", - "google/protobuf/unittest_arena.proto", - "google/protobuf/unittest_custom_options.proto", - "google/protobuf/unittest_drop_unknown_fields.proto", - "google/protobuf/unittest_embed_optimize_for.proto", - "google/protobuf/unittest_empty.proto", - "google/protobuf/unittest_enormous_descriptor.proto", - "google/protobuf/unittest_import.proto", - "google/protobuf/unittest_import_public.proto", - "google/protobuf/unittest_lazy_dependencies.proto", - "google/protobuf/unittest_lazy_dependencies_custom_option.proto", - "google/protobuf/unittest_lazy_dependencies_enum.proto", - "google/protobuf/unittest_lite_imports_nonlite.proto", - "google/protobuf/unittest_mset.proto", - "google/protobuf/unittest_mset_wire_format.proto", - "google/protobuf/unittest_no_field_presence.proto", - "google/protobuf/unittest_no_generic_services.proto", - "google/protobuf/unittest_optimize_for.proto", - "google/protobuf/unittest_preserve_unknown_enum.proto", - "google/protobuf/unittest_preserve_unknown_enum2.proto", - "google/protobuf/unittest_proto3.proto", - "google/protobuf/unittest_proto3_arena.proto", - "google/protobuf/unittest_proto3_arena_lite.proto", - "google/protobuf/unittest_proto3_lite.proto", - "google/protobuf/unittest_proto3_optional.proto", - "google/protobuf/unittest_well_known_types.proto", - "google/protobuf/util/internal/testdata/anys.proto", - "google/protobuf/util/internal/testdata/books.proto", - "google/protobuf/util/internal/testdata/default_value.proto", - "google/protobuf/util/internal/testdata/default_value_test.proto", - "google/protobuf/util/internal/testdata/field_mask.proto", - "google/protobuf/util/internal/testdata/maps.proto", - "google/protobuf/util/internal/testdata/oneofs.proto", - "google/protobuf/util/internal/testdata/proto3.proto", - "google/protobuf/util/internal/testdata/struct.proto", - "google/protobuf/util/internal/testdata/timestamp_duration.proto", - "google/protobuf/util/internal/testdata/wrappers.proto", - "google/protobuf/util/json_format.proto", - "google/protobuf/util/json_format_proto3.proto", - "google/protobuf/util/message_differencer_unittest.proto", -] - -TEST_PROTOS = ["src/" + s for s in RELATIVE_TEST_PROTOS] - -GENERIC_RELATIVE_TEST_PROTOS = [ - "google/protobuf/map_proto2_unittest.proto", - "google/protobuf/map_unittest.proto", - "google/protobuf/unittest.proto", - "google/protobuf/unittest_arena.proto", - "google/protobuf/unittest_custom_options.proto", - "google/protobuf/unittest_drop_unknown_fields.proto", - "google/protobuf/unittest_embed_optimize_for.proto", - "google/protobuf/unittest_empty.proto", - "google/protobuf/unittest_enormous_descriptor.proto", - "google/protobuf/unittest_import.proto", - "google/protobuf/unittest_import_public.proto", - "google/protobuf/unittest_lazy_dependencies.proto", - "google/protobuf/unittest_lazy_dependencies_custom_option.proto", - "google/protobuf/unittest_lazy_dependencies_enum.proto", - "google/protobuf/unittest_lite_imports_nonlite.proto", - "google/protobuf/unittest_mset.proto", - "google/protobuf/unittest_mset_wire_format.proto", - "google/protobuf/unittest_no_field_presence.proto", - "google/protobuf/unittest_no_generic_services.proto", - "google/protobuf/unittest_optimize_for.proto", - "google/protobuf/unittest_preserve_unknown_enum.proto", - "google/protobuf/unittest_preserve_unknown_enum2.proto", - "google/protobuf/unittest_proto3.proto", - "google/protobuf/unittest_proto3_arena.proto", - "google/protobuf/unittest_proto3_arena_lite.proto", - "google/protobuf/unittest_proto3_lite.proto", - "google/protobuf/unittest_proto3_optional.proto", - "google/protobuf/unittest_well_known_types.proto", -] - -GENERIC_TEST_PROTOS = ["src/" + s for s in GENERIC_RELATIVE_TEST_PROTOS] - -proto_library( - name = "generic_test_protos", - srcs = LITE_TEST_PROTOS + GENERIC_TEST_PROTOS, - strip_import_prefix = "src", - visibility = ["//:__subpackages__"], - deps = [ - "//:any_proto", - "//:api_proto", - "//:descriptor_proto", - "//:duration_proto", - "//:empty_proto", - "//:field_mask_proto", - "//:source_context_proto", - "//:struct_proto", - "//:timestamp_proto", - "//:type_proto", - "//:wrappers_proto", - ], -) - -cc_proto_library( - name = "cc_test_protos", - srcs = LITE_TEST_PROTOS + TEST_PROTOS, - include = "src", - default_runtime = ":protobuf", - protoc = ":protoc", - deps = [":cc_wkt_protos"], -) - -COMMON_TEST_SRCS = [ - # AUTOGEN(common_test_srcs) - "src/google/protobuf/arena_test_util.cc", - "src/google/protobuf/map_lite_test_util.cc", - "src/google/protobuf/test_util_lite.cc", - "src/google/protobuf/map_test_util.inc", - "src/google/protobuf/reflection_tester.cc", - "src/google/protobuf/test_util.cc", - "src/google/protobuf/test_util.inc", - "src/google/protobuf/testing/file.cc", - "src/google/protobuf/testing/googletest.cc", -] - -cc_binary( - name = "test_plugin", - testonly = True, - srcs = [ - # AUTOGEN(test_plugin_srcs) - "src/google/protobuf/compiler/mock_code_generator.cc", - "src/google/protobuf/compiler/test_plugin.cc", - "src/google/protobuf/testing/file.cc", - ], - deps = [ - ":protobuf", - ":protoc_lib", - "@com_google_googletest//:gtest", - ], -) - -cc_test( - name = "win32_test", - srcs = ["src/google/protobuf/io/io_win32_unittest.cc"], - tags = [ - "manual", - "windows", - ], - deps = [ - ":protobuf_lite", - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - ], -) - -cc_test( - name = "protobuf_test", - srcs = COMMON_TEST_SRCS + [ - # AUTOGEN(test_srcs) - "src/google/protobuf/any_test.cc", - "src/google/protobuf/arena_unittest.cc", - "src/google/protobuf/arenastring_unittest.cc", - "src/google/protobuf/arenaz_sampler_test.cc", - "src/google/protobuf/compiler/annotation_test_util.cc", - "src/google/protobuf/compiler/command_line_interface_unittest.cc", - "src/google/protobuf/compiler/cpp/bootstrap_unittest.cc", - "src/google/protobuf/compiler/cpp/metadata_test.cc", - "src/google/protobuf/compiler/cpp/move_unittest.cc", - "src/google/protobuf/compiler/cpp/plugin_unittest.cc", - "src/google/protobuf/compiler/cpp/unittest.cc", - "src/google/protobuf/compiler/cpp/unittest.inc", - "src/google/protobuf/compiler/csharp/csharp_bootstrap_unittest.cc", - "src/google/protobuf/compiler/csharp/csharp_generator_unittest.cc", - "src/google/protobuf/compiler/importer_unittest.cc", - "src/google/protobuf/compiler/java/doc_comment_unittest.cc", - "src/google/protobuf/compiler/java/plugin_unittest.cc", - "src/google/protobuf/compiler/mock_code_generator.cc", - "src/google/protobuf/compiler/objectivec/objectivec_helpers_unittest.cc", - "src/google/protobuf/compiler/parser_unittest.cc", - "src/google/protobuf/compiler/python/plugin_unittest.cc", - "src/google/protobuf/compiler/ruby/ruby_generator_unittest.cc", - "src/google/protobuf/descriptor_database_unittest.cc", - "src/google/protobuf/descriptor_unittest.cc", - "src/google/protobuf/drop_unknown_fields_test.cc", - "src/google/protobuf/dynamic_message_unittest.cc", - "src/google/protobuf/extension_set_unittest.cc", - "src/google/protobuf/generated_message_reflection_unittest.cc", - "src/google/protobuf/generated_message_tctable_lite_test.cc", - "src/google/protobuf/inlined_string_field_unittest.cc", - "src/google/protobuf/io/coded_stream_unittest.cc", - "src/google/protobuf/io/io_win32_unittest.cc", - "src/google/protobuf/io/printer_unittest.cc", - "src/google/protobuf/io/tokenizer_unittest.cc", - "src/google/protobuf/io/zero_copy_stream_unittest.cc", - "src/google/protobuf/map_field_test.cc", - "src/google/protobuf/map_test.cc", - "src/google/protobuf/map_test.inc", - "src/google/protobuf/message_unittest.cc", - "src/google/protobuf/message_unittest.inc", - "src/google/protobuf/no_field_presence_test.cc", - "src/google/protobuf/preserve_unknown_enum_test.cc", - "src/google/protobuf/proto3_arena_lite_unittest.cc", - "src/google/protobuf/proto3_arena_unittest.cc", - "src/google/protobuf/proto3_lite_unittest.cc", - "src/google/protobuf/proto3_lite_unittest.inc", - "src/google/protobuf/reflection_ops_unittest.cc", - "src/google/protobuf/repeated_field_reflection_unittest.cc", - "src/google/protobuf/repeated_field_unittest.cc", - "src/google/protobuf/stubs/bytestream_unittest.cc", - "src/google/protobuf/stubs/common_unittest.cc", - "src/google/protobuf/stubs/int128_unittest.cc", - "src/google/protobuf/stubs/status_test.cc", - "src/google/protobuf/stubs/statusor_test.cc", - "src/google/protobuf/stubs/stringpiece_unittest.cc", - "src/google/protobuf/stubs/stringprintf_unittest.cc", - "src/google/protobuf/stubs/structurally_valid_unittest.cc", - "src/google/protobuf/stubs/strutil_unittest.cc", - "src/google/protobuf/stubs/template_util_unittest.cc", - "src/google/protobuf/stubs/time_test.cc", - "src/google/protobuf/text_format_unittest.cc", - "src/google/protobuf/unknown_field_set_unittest.cc", - "src/google/protobuf/util/delimited_message_util_test.cc", - "src/google/protobuf/util/field_comparator_test.cc", - "src/google/protobuf/util/field_mask_util_test.cc", - "src/google/protobuf/util/internal/default_value_objectwriter_test.cc", - "src/google/protobuf/util/internal/json_objectwriter_test.cc", - "src/google/protobuf/util/internal/json_stream_parser_test.cc", - "src/google/protobuf/util/internal/protostream_objectsource_test.cc", - "src/google/protobuf/util/internal/protostream_objectwriter_test.cc", - "src/google/protobuf/util/internal/type_info_test_helper.cc", - "src/google/protobuf/util/json_util_test.cc", - "src/google/protobuf/util/message_differencer_unittest.cc", - "src/google/protobuf/util/time_util_test.cc", - "src/google/protobuf/util/type_resolver_util_test.cc", - "src/google/protobuf/well_known_types_unittest.cc", - "src/google/protobuf/wire_format_unittest.cc", - "src/google/protobuf/wire_format_unittest.inc", - ] + select({ - "//conditions:default": [ - # AUTOGEN(non_msvc_test_srcs) - ], - ":msvc": [], - }), - copts = COPTS + select({ - ":msvc": [], - "//conditions:default": [ - "-Wno-deprecated-declarations", - ], - }), - data = [ - ":test_plugin", - ] + glob([ - "src/google/protobuf/**/*", - # Files for csharp_bootstrap_unittest.cc. - "conformance/**/*", - ]) + glob( - [ - # Files for csharp_bootstrap_unittest.cc. - "csharp/src/**/*", - ], - allow_empty = True, - ), - includes = [ - "src/", - ], +cc_library( + name = "protobuf", + hdrs = glob([ + "src/**/*.h", + "src/**/*.inc", + ]), + copts = COPTS, + include_prefix = "google/protobuf/io", linkopts = LINK_OPTS, + visibility = ["//visibility:public"], deps = [ - ":cc_test_protos", - ":protobuf", - ":protoc_lib", - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - ] + PROTOBUF_DEPS, + "//src/google/protobuf", + "//src/google/protobuf/compiler:importer", + "//src/google/protobuf/util:delimited_message_util", + "//src/google/protobuf/util:differencer", + "//src/google/protobuf/util:field_mask_util", + "//src/google/protobuf/util:json_util", + "//src/google/protobuf/util:time_util", + "//src/google/protobuf/util:type_resolver_util", + ], +) + +# This provides just the header files for use in projects that need to build +# shared libraries for dynamic loading. This target is available until Bazel +# adds native support for such use cases. +# TODO(keveman): Remove this target once the support gets added to Bazel. +alias( + name = "protobuf_headers", + actual = "//src/google/protobuf:protobuf_headers", + visibility = ["//visibility:public"], ) ################################################################################ # Java support ################################################################################ -internal_gen_well_known_protos_java( - name = "gen_well_known_protos_java", - visibility = [ - "//java:__subpackages__", - ], - deps = [proto + "_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()], -) - -internal_gen_well_known_protos_java( - name = "gen_well_known_protos_javalite", - javalite = True, - visibility = [ - "//java:__subpackages__", - ], - deps = [proto + "_proto" for proto in LITE_WELL_KNOWN_PROTO_MAP.keys()], -) - -internal_gen_kt_protos( - name = "gen_well_known_protos_kotlin", - visibility = [ - "//java:__subpackages__", - ], - deps = [proto + "_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()], -) - -internal_gen_kt_protos( - name = "gen_well_known_protos_kotlinlite", - lite = True, - visibility = [ - "//java:__subpackages__", - ], - deps = [proto + "_proto" for proto in LITE_WELL_KNOWN_PROTO_MAP.keys()], -) - alias( name = "protobuf_java", actual = "//java/core", @@ -893,247 +228,58 @@ alias( # Python support ################################################################################ -py_library( - name = "python_srcs", - srcs = glob( - [ - "python/google/protobuf/**/*.py", - ], - ), - imports = ["python"], - srcs_version = "PY2AND3", - visibility = ["@upb//:__subpackages__"], -) - -py_library( - name = "python_test_srcs", - srcs = glob( - [ - "python/google/protobuf/internal/*_test.py", - "python/google/protobuf/internal/test_util.py", - ], - ), - imports = ["python"], - srcs_version = "PY3", - visibility = ["@upb//:__subpackages__"], -) - -cc_binary( - name = "python/google/protobuf/internal/_api_implementation.so", - srcs = ["python/google/protobuf/internal/api_implementation.cc"], - copts = COPTS + [ - "-DPYTHON_PROTO2_CPP_IMPL_V2", - ], - linkshared = 1, - linkstatic = 1, - tags = [ - # Exclude this target from wildcard expansion (//...) because it may - # not even be buildable. It will be built if it is needed according - # to :use_fast_cpp_protos. - # https://docs.bazel.build/versions/master/be/common-definitions.html#common-attributes - "manual", - ], - deps = select({ - "//conditions:default": [], - ":use_fast_cpp_protos": ["//external:python_headers"], - }), -) - -cc_binary( - name = "python/google/protobuf/pyext/_message.so", - srcs = glob([ - "python/google/protobuf/pyext/*.cc", - "python/google/protobuf/pyext/*.h", - ]), - copts = COPTS + [ - "-DGOOGLE_PROTOBUF_HAS_ONEOF=1", - ] + select({ - "//conditions:default": [], - ":allow_oversize_protos": ["-DPROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS=1"], - }), - includes = [ - "python/", - "src/", - ], - linkshared = 1, - linkstatic = 1, - tags = [ - # Exclude this target from wildcard expansion (//...) because it may - # not even be buildable. It will be built if it is needed according - # to :use_fast_cpp_protos. - # https://docs.bazel.build/versions/master/be/common-definitions.html#common-attributes - "manual", - ], - deps = [ - ":protobuf", - ":proto_api", - ] + select({ - "//conditions:default": [], - ":use_fast_cpp_protos": ["//external:python_headers"], - }), -) - -config_setting( - name = "use_fast_cpp_protos", - values = { - "define": "use_fast_cpp_protos=true", - }, - visibility = [ - # Public, but Protobuf only visibility. - "//:__subpackages__", - ], -) - -config_setting( - name = "allow_oversize_protos", - values = { - "define": "allow_oversize_protos=true", - }, - visibility = [ - # Public, but Protobuf only visibility. - "//:__subpackages__", - ], -) - -# Copy the builtin proto files from src/google/protobuf to -# python/google/protobuf. This way, the generated Python sources will be in the -# same directory as the Python runtime sources. This is necessary for the -# modules to be imported correctly since they are all part of the same Python -# package. -internal_copied_filegroup( - name = "protos_python", - srcs = WELL_KNOWN_PROTOS, - dest = "python", - strip_prefix = "src", -) - -# TODO(dzc): Remove this once py_proto_library can have labels in srcs, in -# which case we can simply add :protos_python in srcs. -COPIED_WELL_KNOWN_PROTOS = ["python/" + s[4:] for s in WELL_KNOWN_PROTOS] - -py_proto_library( - name = "well_known_types_py_pb2", - srcs = COPIED_WELL_KNOWN_PROTOS, - include = "python", - default_runtime = "", - protoc = ":protoc", - srcs_version = "PY2AND3", - visibility = ["@upb//:__subpackages__"], -) - -py_library( +alias( name = "protobuf_python", - data = select({ - "//conditions:default": [], - ":use_fast_cpp_protos": [ - ":python/google/protobuf/internal/_api_implementation.so", - ":python/google/protobuf/pyext/_message.so", - ], - }), - deps = [ - ":python_srcs", - ":well_known_types_py_pb2", - ], + actual = "//python:protobuf_python", + visibility = ["//visibility:public"], ) -# Copy the test proto files from src/google/protobuf to -# python/google/protobuf. This way, the generated Python sources will be in the -# same directory as the Python runtime sources. This is necessary for the -# modules to be imported correctly by the tests since they are all part of the -# same Python package. -internal_copied_filegroup( - name = "protos_python_test", - srcs = LITE_TEST_PROTOS + TEST_PROTOS, - dest = "python", - strip_prefix = "src", +alias( + name = "python_srcs", + actual = "//python:python_srcs", + visibility = ["@upb//:__subpackages__"], ) -# TODO(dzc): Remove this once py_proto_library can have labels in srcs, in -# which case we can simply add :protos_python_test in srcs. -COPIED_LITE_TEST_PROTOS = ["python/" + s for s in RELATIVE_LITE_TEST_PROTOS] +alias( + name = "python_test_srcs", + actual = "//python:python_test_srcs", + visibility = ["@upb//:__subpackages__"], +) -COPIED_TEST_PROTOS = ["python/" + s for s in RELATIVE_TEST_PROTOS] +alias( + name = "well_known_types_py_pb2", + actual = "//python:well_known_types_py_pb2", + visibility = ["@upb//:__subpackages__"], +) -py_proto_library( +alias( name = "python_common_test_protos", - srcs = COPIED_LITE_TEST_PROTOS + COPIED_TEST_PROTOS, - include = "python", - default_runtime = "", - protoc = ":protoc", - srcs_version = "PY2AND3", + actual = "//python:python_common_test_protos", visibility = ["//visibility:public"], - deps = [":well_known_types_py_pb2"], ) -py_proto_library( +alias( name = "python_specific_test_protos", - srcs = glob([ - "python/google/protobuf/internal/*.proto", - "python/google/protobuf/internal/import_test_package/*.proto", - ]), - include = "python", - default_runtime = ":protobuf_python", - protoc = ":protoc", - srcs_version = "PY2AND3", + actual = "//python:python_specific_test_protos", visibility = ["//visibility:public"], - deps = [":python_common_test_protos"], -) - -py_library( - name = "python_tests", - srcs = glob( - [ - "python/google/protobuf/internal/*_test.py", - "python/google/protobuf/internal/test_util.py", - "python/google/protobuf/internal/import_test_package/__init__.py", - ], - ), - imports = ["python"], - srcs_version = "PY2AND3", - deps = [ - ":protobuf_python", - ":python_common_test_protos", - ":python_specific_test_protos", - ], -) - -internal_protobuf_py_tests( - name = "python_tests_batch", - data = glob([ - "src/google/protobuf/**/*", - ]), - modules = [ - "descriptor_database_test", - "descriptor_pool_test", - "descriptor_test", - "generator_test", - "json_format_test", - "message_factory_test", - "message_test", - "proto_builder_test", - "reflection_test", - "service_reflection_test", - "symbol_database_test", - "text_encoding_test", - "text_format_test", - "unknown_fields_test", - "wire_format_test", - ], - deps = [":python_tests"], -) - -cc_library( - name = "proto_api", - hdrs = ["python/google/protobuf/proto_api.h"], - visibility = ["//visibility:public"], - deps = [ - "//external:python_headers", - ], ) proto_lang_toolchain( name = "cc_toolchain", - blacklisted_protos = [proto + "_proto" for proto in WELL_KNOWN_PROTO_MAP.keys()], + blacklisted_protos = [ + "@com_google_protobuf//:any_proto", + "@com_google_protobuf//:api_proto", + "@com_google_protobuf//:compiler_plugin_proto", + "@com_google_protobuf//:descriptor_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:empty_proto", + "@com_google_protobuf//:field_mask_proto", + "@com_google_protobuf//:source_context_proto", + "@com_google_protobuf//:struct_proto", + "@com_google_protobuf//:timestamp_proto", + "@com_google_protobuf//:type_proto", + "@com_google_protobuf//:wrappers_proto", + ], command_line = "--cpp_out=$(OUT)", runtime = ":protobuf", visibility = ["//visibility:public"], @@ -1154,137 +300,73 @@ alias( ) ################################################################################ -# Test generated proto support +# Test protos ################################################################################ +alias( + name = "lite_test_protos", + actual = "//src/google/protobuf:lite_test_protos", # proto_library + visibility = ["//:__subpackages__"], +) + +alias( + name = "test_proto_srcs", + actual = "//src/google/protobuf:test_proto_srcs", # filegroup + visibility = ["//:__subpackages__"], +) + +alias( + name = "test_protos", + actual = "//src/google/protobuf:test_protos", # proto_library + visibility = ["//:__subpackages__"], +) + +# Validate generated proto source inputs: + genrule( name = "generated_protos", - srcs = ["src/google/protobuf/unittest_import.proto"], + testonly = 1, + srcs = ["//src/google/protobuf:test_proto_srcs"], outs = ["unittest_gen_import.proto"], - cmd = "cat $(SRCS) | sed 's|google/|src/google/|' > $(OUTS)", + cmd = "cat src/google/protobuf/unittest_import.proto > $@", ) proto_library( name = "generated_protos_proto", - srcs = [ - "src/google/protobuf/unittest_import_public.proto", - "unittest_gen_import.proto", - ], + testonly = 1, + srcs = [":generated_protos"], + deps = ["//src/google/protobuf:generic_test_protos"], ) py_proto_library( name = "generated_protos_py", - srcs = [ - "src/google/protobuf/unittest_import_public.proto", - "unittest_gen_import.proto", - ], + testonly = 1, + srcs = [":generated_protos"], default_runtime = "", protoc = ":protoc", + deps = ["//python:python_common_test_protos"], ) ################################################################################ # Conformance tests ################################################################################ -proto_library( - name = "test_messages_proto2_proto", - srcs = ["src/google/protobuf/test_messages_proto2.proto"], - visibility = ["//visibility:public"], -) - -proto_library( - name = "test_messages_proto3_proto", - srcs = ["src/google/protobuf/test_messages_proto3.proto"], - visibility = ["//visibility:public"], - deps = [ - ":any_proto", - ":duration_proto", - ":field_mask_proto", - ":struct_proto", - ":timestamp_proto", - ":wrappers_proto", - ], -) - -cc_proto_library( - name = "test_messages_proto2_proto_cc", - srcs = ["src/google/protobuf/test_messages_proto2.proto"], -) - -cc_proto_library( - name = "test_messages_proto3_proto_cc", - srcs = ["src/google/protobuf/test_messages_proto3.proto"], - deps = [ - ":cc_wkt_protos", - ], -) - -proto_library( - name = "conformance_proto", - srcs = ["conformance/conformance.proto"], - visibility = ["//visibility:public"], -) - -cc_proto_library( - name = "conformance_proto_cc", - srcs = ["conformance/conformance.proto"], -) - -cc_library( - name = "jsoncpp", - srcs = ["conformance/third_party/jsoncpp/jsoncpp.cpp"], - hdrs = ["conformance/third_party/jsoncpp/json.h"], - includes = ["conformance"], -) - -cc_library( - name = "conformance_test", - srcs = [ - "conformance/conformance_test.cc", - "conformance/conformance_test_runner.cc", - ], - hdrs = [ - "conformance/conformance_test.h", - ], - includes = [ - "conformance", - "src", - ], - deps = [":conformance_proto_cc"], -) - -cc_library( - name = "binary_json_conformance_suite", - srcs = ["conformance/binary_json_conformance_suite.cc"], - hdrs = ["conformance/binary_json_conformance_suite.h"], - deps = [ - ":conformance_test", - ":jsoncpp", - ":test_messages_proto2_proto_cc", - ":test_messages_proto3_proto_cc", - ], -) - -cc_library( - name = "text_format_conformance_suite", - srcs = ["conformance/text_format_conformance_suite.cc"], - hdrs = ["conformance/text_format_conformance_suite.h"], - deps = [ - ":conformance_test", - ":test_messages_proto2_proto_cc", - ":test_messages_proto3_proto_cc", - ], -) - -cc_binary( +alias( name = "conformance_test_runner", - srcs = ["conformance/conformance_test_main.cc"], + actual = "//conformance:conformance_test_runner", + visibility = ["//visibility:public"], +) + +alias( + name = "test_messages_proto2_proto", + actual = "//src/google/protobuf:test_messages_proto2_proto", # proto_library + visibility = ["//visibility:public"], +) + +alias( + name = "test_messages_proto3_proto", + actual = "//src/google/protobuf:test_messages_proto3_proto", # proto_library visibility = ["//visibility:public"], - deps = [ - ":binary_json_conformance_suite", - ":conformance_test", - ":text_format_conformance_suite", - ], ) # TODO: re-enable this test if appropriate, or replace with something that @@ -1306,21 +388,10 @@ cc_binary( # ], # ) -java_proto_library( - name = "java_test_protos", - visibility = ["//java:__subpackages__"], - deps = [":generic_test_protos"], -) - -java_lite_proto_library( - name = "java_lite_test_protos", - visibility = ["//java:__subpackages__"], - deps = [":generic_test_protos"], -) - java_proto_library( name = "test_messages_proto2_java_proto", visibility = [ + "//conformance:__pkg__", "//java:__subpackages__", ], deps = [":test_messages_proto2_proto"], @@ -1329,149 +400,36 @@ java_proto_library( java_proto_library( name = "test_messages_proto3_java_proto", visibility = [ + "//conformance:__pkg__", "//java:__subpackages__", ], deps = [":test_messages_proto3_proto"], ) -java_proto_library( - name = "conformance_java_proto", - visibility = [ - "//java:__subpackages__", - ], - deps = [":conformance_proto"], -) - java_lite_proto_library( name = "test_messages_proto2_java_proto_lite", visibility = [ + "//conformance:__pkg__", "//java:__subpackages__", ], deps = [":test_messages_proto2_proto"], ) -java_lite_proto_library( - name = "conformance_java_proto_lite", - visibility = [ - "//java:__subpackages__", - ], - deps = [":conformance_proto"], -) - java_lite_proto_library( name = "test_messages_proto3_java_proto_lite", visibility = [ + "//conformance:__pkg__", "//java:__subpackages__", ], deps = [":test_messages_proto3_proto"], ) -java_binary( - name = "conformance_java", - srcs = ["conformance/ConformanceJava.java"], - main_class = "ConformanceJava", - visibility = [ - "//java:__subpackages__", - ], - deps = [ - ":conformance_java_proto", - ":test_messages_proto2_java_proto", - ":test_messages_proto3_java_proto", - "//:protobuf_java", - "//:protobuf_java_util", - ], -) - -java_binary( - name = "conformance_java_lite", - srcs = ["conformance/ConformanceJavaLite.java"], - main_class = "ConformanceJavaLite", - visibility = [ - "//java:__subpackages__", - ], - deps = [ - ":conformance_java_proto_lite", - ":test_messages_proto2_java_proto_lite", - ":test_messages_proto3_java_proto_lite", - "//:protobuf_java_util", - "//:protobuf_javalite", - ], -) - -exports_files([ - "conformance/conformance_test_runner.sh", - "conformance/failure_list_java.txt", - "conformance/failure_list_java_lite.txt", - "conformance/text_format_failure_list_java.txt", - "conformance/text_format_failure_list_java_lite.txt", -]) - filegroup( name = "bzl_srcs", srcs = glob(["**/*.bzl"]), visibility = ["//visibility:public"], ) -# Kotlin proto rules - -proto_library( - name = "kt_unittest_lite", - srcs = [ - "src/google/protobuf/map_lite_unittest.proto", - "src/google/protobuf/unittest_import_lite.proto", - "src/google/protobuf/unittest_import_public_lite.proto", - "src/google/protobuf/unittest_lite.proto", - ], - strip_import_prefix = "src", -) - -internal_gen_kt_protos( - name = "gen_kotlin_unittest_lite", - lite = True, - visibility = ["//java:__subpackages__"], - deps = [":kt_unittest_lite"], -) - -proto_library( - name = "kt_unittest", - srcs = [ - "src/google/protobuf/map_proto2_unittest.proto", - "src/google/protobuf/unittest.proto", - "src/google/protobuf/unittest_import.proto", - "src/google/protobuf/unittest_import_public.proto", - ], - strip_import_prefix = "src", -) - -internal_gen_kt_protos( - name = "gen_kotlin_unittest", - visibility = ["//java:__subpackages__"], - deps = [":kt_unittest"], -) - -proto_library( - name = "kt_proto3_unittest", - srcs = [ - "src/google/protobuf/unittest_import.proto", - "src/google/protobuf/unittest_import_public.proto", - "src/google/protobuf/unittest_proto3.proto", - ], - strip_import_prefix = "src", -) - -internal_gen_kt_protos( - name = "gen_kotlin_proto3_unittest_lite", - lite = True, - visibility = ["//java:__subpackages__"], - deps = [":kt_proto3_unittest"], -) - -internal_gen_kt_protos( - name = "gen_kotlin_proto3_unittest", - visibility = ["//java:__subpackages__"], - deps = [":kt_proto3_unittest"], -) - ################################################################################ # Packaging rules ################################################################################ @@ -1519,6 +477,7 @@ pkg_files( "generate_descriptor_proto.sh", "maven_install.json", "update_file_lists.sh", + "//third_party:BUILD.bazel", "//third_party:zlib.BUILD", "//util/python:BUILD.bazel", ], @@ -1526,39 +485,6 @@ pkg_files( visibility = ["//pkg:__pkg__"], ) -# Conformance tests -pkg_files( - name = "conformance_dist_files", - srcs = glob( - ["conformance/**/*"], - exclude = [ - # The following are not in autotools dist: - "conformance/autoload.php", - "conformance/conformance_nodejs.js", - "conformance/conformance_test_runner.sh", - "conformance/failure_list_java_lite.txt", - "conformance/failure_list_jruby.txt", - "conformance/text_format_failure_list_*.txt", - "conformance/update_failure_list.py", - ], - ), - strip_prefix = strip_prefix.from_root(""), - visibility = ["//pkg:__pkg__"], -) - -# C++ runtime -pkg_files( - name = "cpp_dist_files", - srcs = glob( - ["src/**/*"], - exclude = [ - "src/google/protobuf/compiler/objectivec/method_dump.sh", # not in autotools dist - ], - ), - strip_prefix = strip_prefix.from_root(""), - visibility = ["//pkg:__pkg__"], -) - # Additional files for C# pkg_files( name = "csharp_dist_files", @@ -1576,29 +502,3 @@ pkg_files( ], visibility = ["//pkg:__pkg__"], ) - -# Python runtime -pkg_files( - name = "python_dist_files", - srcs = glob([ - "python/google/**/*.proto", - "python/google/**/*.py", - "python/google/protobuf/internal/*.cc", - "python/google/protobuf/pyext/*.cc", - "python/google/protobuf/pyext/*.h", - ]) + [ - "python/MANIFEST.in", - "python/README.md", - "python/google/protobuf/proto_api.h", - "python/google/protobuf/pyext/README", - "python/google/protobuf/python_protobuf.h", - "python/mox.py", - "python/release.sh", - "python/setup.cfg", - "python/setup.py", - "python/stubout.py", - "python/tox.ini", - ], - strip_prefix = strip_prefix.from_root(""), - visibility = ["//pkg:__pkg__"], -) diff --git a/CHANGES.txt b/CHANGES.txt index 4ef9bdbac..3c6bc3510 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,34 +1,66 @@ -Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) - - PHP - * Fix building packaged PHP extension (#9727) - * Fixed composer.json to only advertise compatibility with PHP 7.0+. (#9819) - - Ruby - * Disable the aarch64 build on macOS until it can be fixed. (#9816) - - Other - * Fix versioning issues in 3.20.0 +2022-05-10 version 21.0-rc1 (C++/Java/Python/PHP/Objective-C/C#/Ruby) C++ + * Rename main cmake/CMakeLists.txt to CMakeLists.txt (#9603) + * avoid allocating memory if all extension are cleared (#9345) + * cmake: Call get_filename_component() with DIRECTORY mode instead of PATH mode (#9614) + * Escape GetObject macro inside protoc-generated code (#9739) + * Update CMake configuration to add a dependency on Abseil (#9793) + * Use __constinit only in GCC 12.2 and up (#9936) * Refactor generated message class layout * Optimize tokenizer ParseInteger by removing division * Reserve exactly the right amount of capacity in ExtensionSet::MergeFrom * Parse FLT_MAX correctly when represented in JSON + Java + * Update protobuf_version.bzl to separate protoc and per-language java … (#9900) + * 6x speedup in ArrayEncoder.writeUInt32NotTag + + Python + * Increment python major version to 4 in version.json for python upb (#9926) + * The C extension module for Python has been rewritten to use the upb library. + This is expected to deliver significant performance benefits, especially when + parsing large payloads. There are some minor breaking changes, but these + should not impact most users. For more information see: + https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates + * Due to the breaking changes for Python, the major version number for Python + has been incremented. + * The binary wheel for macOS now supports Apple silicon. + + + PHP + * chore: [PHP] fix phpdoc for MapField keys (#9536) + * [PHP] Remove unnecessary zval initialization (#9600) + * [PHP] fix PHP build system (#9571) + * Fix building packaged PHP extension (#9727) + * fix: reserve "ReadOnly" keyword for PHP 8.1 and add compatibility (#9633) + * fix: phpdoc syntax for repeatedfield parameters (#9784) + * fix: phpdoc for repeatedfield (#9783) + * Change enum string name for reserved words (#9780) + * Fixed composer.json to only advertise compatibility with PHP 7.0+. (#9819) + + Ruby + * Allow pre-compiled binaries for ruby 3.1.0 (#9566) + * Implement `respond_to?` in RubyMessage (#9677) + * [Ruby] Fix RepeatedField#last, #first inconsistencies (#9722) + * Do not use range based UTF-8 validation in truffleruby (#9769) + * Improve range handling logic of `RepeatedField` (#9799) + * Disable the aarch64 build on macOS until it can be fixed. (#9816) + + Other + * [Kotlin] remove redundant public modifiers for compiled code (#9642) + * [C#] Update GetExtension to support getting typed value (#9655) + * Fix invalid dependency manifest when using `descriptor_set_out` (#9647) + * Fix C# generator handling of a field named "none" in a oneof (#9636) + * Add initial version.json file for 21-dev (#9840) + * Remove duplicate java generated code (#9909) + * Fix versioning issues in 3.20.0 + Compiler * Protoc outputs the list of suggested field numbers when invalid field numbers are specified in the .proto file. * Require package names to be less than 512 bytes in length - Java - * 6x speedup in ArrayEncoder.writeUInt32NotTag - - Python - * Added UnknownFieldSet(message) for pure Python. The old - message.UnknownFields() will be deprecated after UnknownFieldSet(message) is - added for cpp extension. - 2022-04-21 version 3.20.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript) PHP diff --git a/Makefile.am b/Makefile.am index 98d45b613..7f064294b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -993,6 +993,7 @@ php_EXTRA_DIST= \ # Note: please keep this in sync with the python_dist_files rule in BUILD.bazel. python_EXTRA_DIST= \ + python/BUILD.bazel \ python/MANIFEST.in \ python/google/__init__.py \ python/google/protobuf/__init__.py \ @@ -1107,6 +1108,7 @@ python_EXTRA_DIST= \ python/google/protobuf/text_format.py \ python/google/protobuf/unknown_fields.py \ python/google/protobuf/util/__init__.py \ + python/internal.bzl \ python/release.sh \ python/mox.py \ python/setup.cfg \ @@ -1205,15 +1207,20 @@ EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ BUILD.bazel \ WORKSPACE \ CMakeLists.txt \ - cmake/abseil-cpp.cmake \ + build_defs/BUILD.bazel \ + build_defs/cc_proto_blacklist_test.bzl \ + build_defs/compiler_config_setting.bzl \ + build_defs/cpp_opts.bzl \ + build_files_updated_unittest.sh \ cmake/CMakeLists.txt \ cmake/README.md \ + cmake/abseil-cpp.cmake \ cmake/conformance.cmake \ cmake/examples.cmake \ cmake/extract_includes.bat.in \ cmake/install.cmake \ - cmake/libprotobuf.cmake \ cmake/libprotobuf-lite.cmake \ + cmake/libprotobuf.cmake \ cmake/libprotoc.cmake \ cmake/protobuf-config-version.cmake.in \ cmake/protobuf-config.cmake.in \ @@ -1224,9 +1231,6 @@ EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ cmake/protoc.cmake \ cmake/tests.cmake \ cmake/version.rc.in \ - compiler_config_setting.bzl \ - build_files_updated_unittest.sh \ - cc_proto_blacklist_test.bzl \ csharp/BUILD.bazel \ editors/README.txt \ editors/proto.vim \ @@ -1252,7 +1256,8 @@ EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ examples/list_people.dart \ examples/list_people.py \ examples/pubspec.yaml \ - internal.bzl \ + conformance/BUILD.bazel \ + conformance/defs.bzl \ maven_install.json \ php/BUILD.bazel \ protobuf.bzl \ diff --git a/Protobuf-C++.podspec b/Protobuf-C++.podspec index 780445961..9d8815a8c 100644 --- a/Protobuf-C++.podspec +++ b/Protobuf-C++.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Protobuf-C++' - s.version = '3.20.1' + s.version = '3.21.0-rc1' s.summary = 'Protocol Buffers v3 runtime library for C++.' s.homepage = 'https://github.com/google/protobuf' s.license = 'BSD-3-Clause' diff --git a/Protobuf.podspec b/Protobuf.podspec index 3a5132d09..37d7fc9a9 100644 --- a/Protobuf.podspec +++ b/Protobuf.podspec @@ -5,7 +5,7 @@ # dependent projects use the :git notation to refer to the library. Pod::Spec.new do |s| s.name = 'Protobuf' - s.version = '3.20.1' + s.version = '3.21.0-rc1' s.summary = 'Protocol Buffers v.3 runtime library for Objective-C.' s.homepage = 'https://github.com/protocolbuffers/protobuf' s.license = 'BSD-3-Clause' diff --git a/benchmarks/BUILD.bazel b/benchmarks/BUILD.bazel index 83b2c7ee5..36f6283f6 100644 --- a/benchmarks/BUILD.bazel +++ b/benchmarks/BUILD.bazel @@ -75,7 +75,6 @@ pkg_files( ], exclude = [ "__init__.py", # not in autotools dist - "BUILD.bazel", "go/*", ], ), diff --git a/benchmarks/cpp/BUILD.bazel b/benchmarks/cpp/BUILD.bazel index ba93f641e..7a3d3ba63 100644 --- a/benchmarks/cpp/BUILD.bazel +++ b/benchmarks/cpp/BUILD.bazel @@ -17,7 +17,10 @@ cc_binary( pkg_files( name = "dist_files", - srcs = ["cpp_benchmark.cc"], + srcs = [ + "BUILD.bazel", + "cpp_benchmark.cc", + ], strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/benchmarks/datasets/google_message1/proto2/BUILD.bazel b/benchmarks/datasets/google_message1/proto2/BUILD.bazel index e3159327b..30caed574 100644 --- a/benchmarks/datasets/google_message1/proto2/BUILD.bazel +++ b/benchmarks/datasets/google_message1/proto2/BUILD.bazel @@ -46,7 +46,7 @@ java_proto_library( pkg_files( name = "dist_files", - srcs = glob(["*.proto"]), + srcs = glob(["*"]), strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/benchmarks/datasets/google_message1/proto3/BUILD.bazel b/benchmarks/datasets/google_message1/proto3/BUILD.bazel index 2628d020a..0dc59d144 100644 --- a/benchmarks/datasets/google_message1/proto3/BUILD.bazel +++ b/benchmarks/datasets/google_message1/proto3/BUILD.bazel @@ -46,7 +46,7 @@ java_proto_library( pkg_files( name = "dist_files", - srcs = glob(["*.proto"]), + srcs = glob(["*"]), strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/benchmarks/datasets/google_message2/BUILD.bazel b/benchmarks/datasets/google_message2/BUILD.bazel index a3208d08c..f3d66a25c 100644 --- a/benchmarks/datasets/google_message2/BUILD.bazel +++ b/benchmarks/datasets/google_message2/BUILD.bazel @@ -46,7 +46,7 @@ java_proto_library( pkg_files( name = "dist_files", - srcs = glob(["*.proto"]), + srcs = glob(["*"]), strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/benchmarks/datasets/google_message3/BUILD.bazel b/benchmarks/datasets/google_message3/BUILD.bazel index 130c8749b..a729e50fb 100644 --- a/benchmarks/datasets/google_message3/BUILD.bazel +++ b/benchmarks/datasets/google_message3/BUILD.bazel @@ -52,7 +52,7 @@ java_proto_library( pkg_files( name = "dist_files", - srcs = glob(["*.proto"]), + srcs = glob(["*"]), strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/benchmarks/datasets/google_message4/BUILD.bazel b/benchmarks/datasets/google_message4/BUILD.bazel index 9c7190d93..33de09378 100644 --- a/benchmarks/datasets/google_message4/BUILD.bazel +++ b/benchmarks/datasets/google_message4/BUILD.bazel @@ -47,7 +47,7 @@ java_proto_library( pkg_files( name = "dist_files", - srcs = glob(["*.proto"]), + srcs = glob(["*"]), strip_prefix = strip_prefix.from_root(""), visibility = ["//benchmarks:__pkg__"], ) diff --git a/build_defs/BUILD.bazel b/build_defs/BUILD.bazel new file mode 100644 index 000000000..29a1f1256 --- /dev/null +++ b/build_defs/BUILD.bazel @@ -0,0 +1,159 @@ +# Internal Starlark definitions for Protobuf. + +load("@rules_cc//cc:defs.bzl", starlark_cc_proto_library = "cc_proto_library") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load(":cc_proto_blacklist_test.bzl", "cc_proto_blacklist_test") +load(":compiler_config_setting.bzl", "create_compiler_config_setting") + +package( + default_visibility = [ + # Public, but Protobuf only visibility. + "//:__subpackages__", + ], +) + +create_compiler_config_setting( + name = "config_msvc", + value = "msvc-cl", +) + +# Android NDK builds can specify different crosstool_top flags to choose which +# STL they use for C++. We need these multiple variants to catch all of those +# versions of crosstool_top and reliably detect Android. +# +# For more info on the various crosstool_tops used by NDK Bazel builds, see: +# https://docs.bazel.build/versions/master/android-ndk.html#configuring-the-stl + +config_setting( + name = "config_android", + values = { + "crosstool_top": "//external:android/crosstool", + }, +) + +config_setting( + name = "config_android-stlport", + values = { + "crosstool_top": "@androidndk//:toolchain-stlport", + }, +) + +config_setting( + name = "config_android-libcpp", + values = { + "crosstool_top": "@androidndk//:toolchain-libcpp", + }, +) + +config_setting( + name = "config_android-gnu-libstdcpp", + values = { + "crosstool_top": "@androidndk//:toolchain-gnu-libstdcpp", + }, +) + +config_setting( + name = "config_android-default", + values = { + "crosstool_top": "@androidndk//:default_crosstool", + }, +) + +# Internal testing: + +starlark_cc_proto_library( + name = "any_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:any_proto"], +) + +starlark_cc_proto_library( + name = "api_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:api_proto"], +) + +starlark_cc_proto_library( + name = "compiler_plugin_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:compiler_plugin_proto"], +) + +starlark_cc_proto_library( + name = "descriptor_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:descriptor_proto"], +) + +starlark_cc_proto_library( + name = "duration_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:duration_proto"], +) + +starlark_cc_proto_library( + name = "empty_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:empty_proto"], +) + +starlark_cc_proto_library( + name = "field_mask_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:field_mask_proto"], +) + +starlark_cc_proto_library( + name = "source_context_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:source_context_proto"], +) + +starlark_cc_proto_library( + name = "struct_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:struct_proto"], +) + +starlark_cc_proto_library( + name = "timestamp_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:timestamp_proto"], +) + +starlark_cc_proto_library( + name = "type_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:type_proto"], +) + +starlark_cc_proto_library( + name = "wrappers_cc_proto", + visibility = ["//visibility:private"], + deps = ["//:wrappers_proto"], +) + +cc_proto_blacklist_test( + name = "cc_proto_blacklist_test", + deps = [ + ":any_cc_proto", + ":api_cc_proto", + ":compiler_plugin_cc_proto", + ":descriptor_cc_proto", + ":duration_cc_proto", + ":empty_cc_proto", + ":field_mask_cc_proto", + ":source_context_cc_proto", + ":struct_cc_proto", + ":timestamp_cc_proto", + ":type_cc_proto", + ":wrappers_cc_proto", + ], +) + +pkg_files( + name = "dist_files", + srcs = glob(["*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//pkg:__pkg__"], +) diff --git a/cc_proto_blacklist_test.bzl b/build_defs/cc_proto_blacklist_test.bzl similarity index 92% rename from cc_proto_blacklist_test.bzl rename to build_defs/cc_proto_blacklist_test.bzl index df54293cb..260abdef4 100644 --- a/cc_proto_blacklist_test.bzl +++ b/build_defs/cc_proto_blacklist_test.bzl @@ -14,14 +14,14 @@ def _cc_proto_blacklist_test_impl(ctx): env = unittest.begin(ctx) for dep in ctx.attr.deps: - files = len(dep.files.to_list()) + files = dep.files.to_list() asserts.equals( env, - 0, + [], files, "Expected that target '{}' does not provide files, got {}".format( dep.label, - files, + len(files), ), ) diff --git a/compiler_config_setting.bzl b/build_defs/compiler_config_setting.bzl similarity index 100% rename from compiler_config_setting.bzl rename to build_defs/compiler_config_setting.bzl diff --git a/build_defs/cpp_opts.bzl b/build_defs/cpp_opts.bzl new file mode 100644 index 000000000..f868c4ae5 --- /dev/null +++ b/build_defs/cpp_opts.bzl @@ -0,0 +1,40 @@ +# C++ compile/link options for Protobuf. + +COPTS = select({ + "//build_defs:config_msvc": [ + "/wd4065", # switch statement contains 'default' but no 'case' labels + "/wd4244", # 'conversion' conversion from 'type1' to 'type2', possible loss of data + "/wd4251", # 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2' + "/wd4267", # 'var' : conversion from 'size_t' to 'type', possible loss of data + "/wd4305", # 'identifier' : truncation from 'type1' to 'type2' + "/wd4307", # 'operator' : integral constant overflow + "/wd4309", # 'conversion' : truncation of constant value + "/wd4334", # 'operator' : result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?) + "/wd4355", # 'this' : used in base member initializer list + "/wd4506", # no definition for inline function 'function' + "/wd4800", # 'type' : forcing value to bool 'true' or 'false' (performance warning) + "/wd4996", # The compiler encountered a deprecated declaration. + ], + "//conditions:default": [ + "-DHAVE_ZLIB", + "-Woverloaded-virtual", + "-Wno-sign-compare", + ], +}) + +# Android and MSVC builds do not need to link in a separate pthread library. +LINK_OPTS = select({ + "//build_defs:config_android": [], + "//build_defs:config_android-stlport": [], + "//build_defs:config_android-libcpp": [], + "//build_defs:config_android-gnu-libstdcpp": [], + "//build_defs:config_android-default": [], + "//build_defs:config_msvc": [ + # Suppress linker warnings about files with no symbols defined. + "-ignore:4221", + ], + "//conditions:default": [ + "-lpthread", + "-lm", + ], +}) diff --git a/cmake/abseil-cpp.cmake b/cmake/abseil-cpp.cmake index 10e7e5c00..8aff9d6a2 100644 --- a/cmake/abseil-cpp.cmake +++ b/cmake/abseil-cpp.cmake @@ -1,4 +1,7 @@ -if(protobuf_ABSL_PROVIDER STREQUAL "module") +if(TARGET absl::strings) + # If absl is included already, skip including it. + # (https://github.com/grpc/grpc/issues/29608) +elseif(protobuf_ABSL_PROVIDER STREQUAL "module") if(NOT ABSL_ROOT_DIR) set(ABSL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp) endif() diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in index 061b240ae..129282921 100644 --- a/cmake/extract_includes.bat.in +++ b/cmake/extract_includes.bat.in @@ -51,6 +51,7 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\descriptor_database.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\duration.pb.h" include\google\protobuf\duration.pb.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\dynamic_message.h" include\google\protobuf\dynamic_message.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\empty.pb.h" include\google\protobuf\empty.pb.h +copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\endian.h" include\google\protobuf\endian.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\explicitly_constructed.h" include\google\protobuf\explicitly_constructed.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set.h" include\google\protobuf\extension_set.h copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set_inl.h" include\google\protobuf\extension_set_inl.h diff --git a/cmake/libprotobuf-lite.cmake b/cmake/libprotobuf-lite.cmake index 04a4ba022..1e4444e20 100644 --- a/cmake/libprotobuf-lite.cmake +++ b/cmake/libprotobuf-lite.cmake @@ -115,7 +115,7 @@ if(protobuf_BUILD_SHARED_LIBS) endif() set_target_properties(libprotobuf-lite PROPERTIES VERSION ${protobuf_VERSION} - SOVERSION 31 + SOVERSION 32 OUTPUT_NAME ${LIB_PREFIX}protobuf-lite DEBUG_POSTFIX "${protobuf_DEBUG_POSTFIX}") add_library(protobuf::libprotobuf-lite ALIAS libprotobuf-lite) diff --git a/cmake/libprotobuf.cmake b/cmake/libprotobuf.cmake index 9f3bdb22f..e597959d2 100644 --- a/cmake/libprotobuf.cmake +++ b/cmake/libprotobuf.cmake @@ -131,7 +131,7 @@ if(protobuf_BUILD_SHARED_LIBS) endif() set_target_properties(libprotobuf PROPERTIES VERSION ${protobuf_VERSION} - SOVERSION 31 + SOVERSION 32 OUTPUT_NAME ${LIB_PREFIX}protobuf DEBUG_POSTFIX "${protobuf_DEBUG_POSTFIX}") add_library(protobuf::libprotobuf ALIAS libprotobuf) diff --git a/cmake/libprotoc.cmake b/cmake/libprotoc.cmake index 5883e612a..7506b0220 100644 --- a/cmake/libprotoc.cmake +++ b/cmake/libprotoc.cmake @@ -133,7 +133,7 @@ endif() set_target_properties(libprotoc PROPERTIES COMPILE_DEFINITIONS LIBPROTOC_EXPORTS VERSION ${protobuf_VERSION} - SOVERSION 31 + SOVERSION 32 OUTPUT_NAME ${LIB_PREFIX}protoc DEBUG_POSTFIX "${protobuf_DEBUG_POSTFIX}") add_library(protobuf::libprotoc ALIAS libprotoc) diff --git a/configure.ac b/configure.ac index d5b1f707b..7cf520012 100644 --- a/configure.ac +++ b/configure.ac @@ -17,7 +17,7 @@ AC_PREREQ(2.59) # In the SVN trunk, the version should always be the next anticipated release # version with the "-pre" suffix. (We used to use "-SNAPSHOT" but this pushed # the size of one file name in the dist tarfile over the 99-char limit.) -AC_INIT([Protocol Buffers],[3.20.1],[protobuf@googlegroups.com],[protobuf]) +AC_INIT([Protocol Buffers],[3.21.0-rc-1],[protobuf@googlegroups.com],[protobuf]) AM_MAINTAINER_MODE([enable]) diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel new file mode 100644 index 000000000..fef036600 --- /dev/null +++ b/conformance/BUILD.bazel @@ -0,0 +1,183 @@ +# Conformance testing for Protobuf. + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library") +load( + "@rules_pkg//:mappings.bzl", + "pkg_attributes", + "pkg_filegroup", + "pkg_files", + "strip_prefix", +) + +exports_files([ + "conformance_test_runner.sh", + "failure_list_java.txt", + "failure_list_java_lite.txt", + "text_format_failure_list_java.txt", + "text_format_failure_list_java_lite.txt", +]) + +cc_proto_library( + name = "test_messages_proto2_proto_cc", + deps = ["//:test_messages_proto2_proto"], +) + +cc_proto_library( + name = "test_messages_proto3_proto_cc", + deps = ["//:test_messages_proto3_proto"], +) + +proto_library( + name = "conformance_proto", + srcs = ["conformance.proto"], + visibility = ["//visibility:public"], +) + +cc_proto_library( + name = "conformance_proto_cc", + deps = [":conformance_proto"], +) + +java_proto_library( + name = "conformance_java_proto", + visibility = [ + "//java:__subpackages__", + ], + deps = [":conformance_proto"], +) + +java_lite_proto_library( + name = "conformance_java_proto_lite", + visibility = [ + "//java:__subpackages__", + ], + deps = [":conformance_proto"], +) + +cc_library( + name = "jsoncpp", + srcs = ["third_party/jsoncpp/jsoncpp.cpp"], + hdrs = ["third_party/jsoncpp/json.h"], + includes = ["."], +) + +cc_library( + name = "conformance_test", + srcs = [ + "conformance_test.cc", + "conformance_test_runner.cc", + ], + hdrs = [ + "conformance_test.h", + ], + includes = ["."], + deps = [":conformance_proto_cc"], +) + +cc_library( + name = "binary_json_conformance_suite", + srcs = ["binary_json_conformance_suite.cc"], + hdrs = ["binary_json_conformance_suite.h"], + deps = [ + ":conformance_test", + ":jsoncpp", + ":test_messages_proto2_proto_cc", + ":test_messages_proto3_proto_cc", + ], +) + +cc_library( + name = "text_format_conformance_suite", + srcs = ["text_format_conformance_suite.cc"], + hdrs = ["text_format_conformance_suite.h"], + deps = [ + ":conformance_test", + ":test_messages_proto2_proto_cc", + ":test_messages_proto3_proto_cc", + ], +) + +cc_binary( + name = "conformance_test_runner", + srcs = ["conformance_test_main.cc"], + visibility = ["//visibility:public"], + deps = [ + ":binary_json_conformance_suite", + ":conformance_test", + ":text_format_conformance_suite", + ], +) + +java_binary( + name = "conformance_java", + srcs = ["ConformanceJava.java"], + main_class = "ConformanceJava", + visibility = [ + "//java:__subpackages__", + ], + deps = [ + ":conformance_java_proto", + "//:protobuf_java", + "//:protobuf_java_util", + "//:test_messages_proto2_java_proto", + "//:test_messages_proto3_java_proto", + ], +) + +java_binary( + name = "conformance_java_lite", + srcs = ["ConformanceJavaLite.java"], + main_class = "ConformanceJavaLite", + visibility = [ + "//java:__subpackages__", + ], + deps = [ + ":conformance_java_proto_lite", + "//:protobuf_java_util", + "//:protobuf_javalite", + "//:test_messages_proto2_java_proto_lite", + "//:test_messages_proto3_java_proto_lite", + ], +) + +filegroup( + name = "all_files", + srcs = glob(["**/*"]), + visibility = ["//src/google/protobuf/compiler/csharp:__pkg__"], +) + +pkg_files( + name = "dist_files", + srcs = glob( + ["**/*"], + exclude = [ + # Handled by dist_scripts: + "conformance_test_runner.sh", + + # The following are not in autotools dist: + "autoload.php", + "conformance_nodejs.js", + "failure_list_jruby.txt", + "update_failure_list.py", + ], + ), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//pkg:__pkg__"], +) + +pkg_files( + name = "dist_scripts", + srcs = ["conformance_test_runner.sh"], + attributes = pkg_attributes(mode = "0555"), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//pkg:__pkg__"], +) + +pkg_filegroup( + name = "all_dist_files", + srcs = [ + ":dist_files", + ":dist_scripts", + ], + visibility = ["//pkg:__pkg__"], +) diff --git a/conformance/conformance_test_runner.sh b/conformance/conformance_test_runner.sh index e9b2e4fff..3149f82fb 100755 --- a/conformance/conformance_test_runner.sh +++ b/conformance/conformance_test_runner.sh @@ -40,7 +40,7 @@ while [[ -n "$@" ]]; do esac done -conformance_test_runner=$(rlocation com_google_protobuf/conformance_test_runner) +conformance_test_runner=$(rlocation com_google_protobuf/conformance/conformance_test_runner) conformance_testee=$(rlocation $TESTEE) args=(--enforce_recommended) diff --git a/internal.bzl b/conformance/defs.bzl similarity index 83% rename from internal.bzl rename to conformance/defs.bzl index a281418c9..905953a11 100644 --- a/internal.bzl +++ b/conformance/defs.bzl @@ -12,9 +12,9 @@ def conformance_test(name, testee, failure_list = None, text_format_failure_list native.sh_test( name = name, - srcs = ["//:conformance/conformance_test_runner.sh"], + srcs = ["//conformance:conformance_test_runner.sh"], data = [testee] + failure_lists + [ - "//:conformance_test_runner", + "//conformance:conformance_test_runner", ], args = args, deps = [ @@ -22,8 +22,7 @@ def conformance_test(name, testee, failure_list = None, text_format_failure_list ], ) - def _strip_bazel(testee): if testee.startswith("//"): - testee = testee.replace("//", "com_google_protobuf") + testee = testee.replace("//", "com_google_protobuf/") return testee.replace(":", "/") diff --git a/csharp/BUILD.bazel b/csharp/BUILD.bazel index 2c150841b..1b381b3a3 100644 --- a/csharp/BUILD.bazel +++ b/csharp/BUILD.bazel @@ -4,6 +4,25 @@ load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +filegroup( + name = "wkt_cs_srcs", + data = [ + "src/Google.Protobuf.Conformance/Conformance.cs", + "src/Google.Protobuf/Reflection/Descriptor.cs", + "src/Google.Protobuf/WellKnownTypes/Any.cs", + "src/Google.Protobuf/WellKnownTypes/Api.cs", + "src/Google.Protobuf/WellKnownTypes/Duration.cs", + "src/Google.Protobuf/WellKnownTypes/Empty.cs", + "src/Google.Protobuf/WellKnownTypes/FieldMask.cs", + "src/Google.Protobuf/WellKnownTypes/SourceContext.cs", + "src/Google.Protobuf/WellKnownTypes/Struct.cs", + "src/Google.Protobuf/WellKnownTypes/Timestamp.cs", + "src/Google.Protobuf/WellKnownTypes/Type.cs", + "src/Google.Protobuf/WellKnownTypes/Wrappers.cs", + ], + visibility = ["//src/google/protobuf/compiler/csharp:__pkg__"], +) + pkg_files( name = "dist_files", srcs = glob([ diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index 1d34a5e2d..25c0609b4 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ Google Protocol Buffers tools Tools for Protocol Buffers - Google's data interchange format. See project site for more info. - 3.20.1 + 3.21.0-rc1 Google Inc. protobuf-packages https://github.com/protocolbuffers/protobuf/blob/main/LICENSE diff --git a/csharp/src/AddressBook/Addressbook.cs b/csharp/src/AddressBook/Addressbook.cs index 484a2296a..1ff0f3b6c 100644 --- a/csharp/src/AddressBook/Addressbook.cs +++ b/csharp/src/AddressBook/Addressbook.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: addressbook.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs b/csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs index 412440417..fe21d0c40 100644 --- a/csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs +++ b/csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: datasets/google_message1/proto3/benchmark_message1_proto3.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Benchmarks/Benchmarks.cs b/csharp/src/Google.Protobuf.Benchmarks/Benchmarks.cs index faa37b050..a05802da5 100644 --- a/csharp/src/Google.Protobuf.Benchmarks/Benchmarks.cs +++ b/csharp/src/Google.Protobuf.Benchmarks/Benchmarks.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: benchmarks.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmarkMessages.cs b/csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmarkMessages.cs index df427e802..e2eb2e8b9 100644 --- a/csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmarkMessages.cs +++ b/csharp/src/Google.Protobuf.Benchmarks/WrapperBenchmarkMessages.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: wrapper_benchmark_messages.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Conformance/Conformance.cs b/csharp/src/Google.Protobuf.Conformance/Conformance.cs index 0a3d5c604..79977a0ca 100644 --- a/csharp/src/Google.Protobuf.Conformance/Conformance.cs +++ b/csharp/src/Google.Protobuf.Conformance/Conformance.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: conformance.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs b/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs index 4791b7325..57e59a9ac 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/MapUnittestProto3.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: map_unittest_proto3.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs index 94f089c3f..35f2b8a15 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto2.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/test_messages_proto2.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs index 5a2aa3bd5..520216fdf 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/TestMessagesProto3.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/test_messages_proto3.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs b/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs index 706b7c473..c1f43ce0f 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/Unittest.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestCustomOptionsProto3.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestCustomOptionsProto3.cs index 305dfe188..ea7a936af 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestCustomOptionsProto3.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestCustomOptionsProto3.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest_custom_options_proto3.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImport.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImport.cs index 4f40564eb..de5f0c24d 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImport.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImport.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest_import.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportProto3.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportProto3.cs index a3ce7a225..d661c2d47 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportProto3.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportProto3.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest_import_proto3.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportPublic.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportPublic.cs index b1e0f30ad..f8a0e55fb 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportPublic.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportPublic.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest_import_public.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportPublicProto3.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportPublicProto3.cs index dde8b9bda..f664c7ba9 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportPublicProto3.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestImportPublicProto3.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest_import_public_proto3.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936A.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936A.cs index 56fde4f00..e7e2b4275 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936A.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936A.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest_issue6936_a.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936B.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936B.cs index 1ac4a6729..21232fbd6 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936B.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936B.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest_issue6936_b.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936C.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936C.cs index d7789b87a..cb53b7a19 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936C.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssue6936C.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest_issue6936_c.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssues.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssues.cs index 344010551..3fe2dd68a 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssues.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestIssues.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest_issues.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestProto3.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestProto3.cs index 1af0d9e3f..ddb73d779 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestProto3.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestProto3.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest_proto3.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestProto3Optional.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestProto3Optional.cs index a41b890fc..4502ffaa1 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestProto3Optional.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestProto3Optional.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/unittest_proto3_optional.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestSelfreferentialOptions.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestSelfreferentialOptions.cs index 7a8f72d4b..e1118c96c 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestSelfreferentialOptions.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestSelfreferentialOptions.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: unittest_selfreferential_options.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs index 58e866e76..3ec8d3556 100644 --- a/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs +++ b/csharp/src/Google.Protobuf.Test.TestProtos/UnittestWellKnownTypes.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/unittest_well_known_types.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf.Test/FieldMaskTreeTest.cs b/csharp/src/Google.Protobuf.Test/FieldMaskTreeTest.cs index f71744a87..c7fbd1373 100644 --- a/csharp/src/Google.Protobuf.Test/FieldMaskTreeTest.cs +++ b/csharp/src/Google.Protobuf.Test/FieldMaskTreeTest.cs @@ -432,5 +432,89 @@ namespace Google.Protobuf Assert.IsNotNull(destination.Payload); } + [Test] + public void MergeWrapperFieldsWithNonNullFieldsInSource() + { + // Instantiate a destination with wrapper-based field types. + var destination = new TestWellKnownTypes() + { + StringField = "Hello", + Int32Field = 12, + Int64Field = 24, + BoolField = true, + }; + + // Set up a targeted update. + var source = new TestWellKnownTypes() + { + StringField = "Hi", + Int64Field = 240 + }; + + Merge(new FieldMaskTree().AddFieldPath("string_field").AddFieldPath("int64_field"), + source, + destination, + new FieldMask.MergeOptions(), + false); + + // Make sure the targeted fields changed. + Assert.AreEqual("Hi", destination.StringField); + Assert.AreEqual(240, destination.Int64Field); + + // Prove that non-targeted fields stay intact... + Assert.AreEqual(12, destination.Int32Field); + Assert.IsTrue(destination.BoolField); + + // ...including default values which were not explicitly set in the destination object. + Assert.IsNull(destination.FloatField); + } + + [Test] + [TestCase(false, "Hello", 24)] + [TestCase(true, null, null)] + public void MergeWrapperFieldsWithNullFieldsInSource( + bool replaceMessageFields, + string expectedStringValue, + long? expectedInt64Value) + { + // Instantiate a destination with wrapper-based field types. + var destination = new TestWellKnownTypes() + { + StringField = "Hello", + Int32Field = 12, + Int64Field = 24, + BoolField = true, + }; + + // Set up a targeted update with null valued fields. + var source = new TestWellKnownTypes() + { + StringField = null, + Int64Field = null + }; + + Merge(new FieldMaskTree().AddFieldPath("string_field").AddFieldPath("int64_field"), + source, + destination, + new FieldMask.MergeOptions() + { + ReplaceMessageFields = replaceMessageFields + }, + false); + + // Make sure the targeted fields changed according to our expectations, depending on the value of ReplaceMessageFields. + // When ReplaceMessageFields is false, the null values are not applied to the destination, because, although wrapped types + // are semantically primitives, FieldMaskTree.Merge still treats them as message types in order to maintain consistency with other Protobuf + // libraries such as Java and C++. + Assert.AreEqual(expectedStringValue, destination.StringField); + Assert.AreEqual(expectedInt64Value, destination.Int64Field); + + // Prove that non-targeted fields stay intact... + Assert.AreEqual(12, destination.Int32Field); + Assert.IsTrue(destination.BoolField); + + // ...including default values which were not explicitly set in the destination object. + Assert.IsNull(destination.FloatField); + } } } diff --git a/csharp/src/Google.Protobuf.Test/WellKnownTypes/AnyTest.cs b/csharp/src/Google.Protobuf.Test/WellKnownTypes/AnyTest.cs index c05cb0853..771f1f41b 100644 --- a/csharp/src/Google.Protobuf.Test/WellKnownTypes/AnyTest.cs +++ b/csharp/src/Google.Protobuf.Test/WellKnownTypes/AnyTest.cs @@ -30,8 +30,11 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endregion +using Google.Protobuf.Reflection; using Google.Protobuf.TestProtos; using NUnit.Framework; +using System.Linq; +using UnitTest.Issues.TestProtos; namespace Google.Protobuf.WellKnownTypes { @@ -148,10 +151,34 @@ namespace Google.Protobuf.WellKnownTypes Assert.False(any.Is(TestOneof.Descriptor)); } + [Test] public void IsRightType() { var any = Any.Pack(SampleMessages.CreateFullTestAllTypes()); Assert.True(any.Is(TestAllTypes.Descriptor)); } + + [Test] + public void Unpack_TypeRegistry() + { + var messages = new IMessage[] + { + SampleMessages.CreateFullTestAllTypes(), + new TestWellKnownTypes { BoolField = true }, + new MoreString { Data = { "x" } }, + new MoreBytes { Data = ByteString.CopyFromUtf8("xyz") }, + new ReservedNames { Descriptor_ = 10 } + }; + var anyMessages = messages.Select(Any.Pack); + + // The type registry handles the first four of the packed messages, but not the final one. + var registry = TypeRegistry.FromFiles( + UnittestWellKnownTypesReflection.Descriptor, + UnittestProto3Reflection.Descriptor); + var unpacked = anyMessages.Select(any => any.Unpack(registry)).ToList(); + var expected = (IMessage[]) messages.Clone(); + expected[4] = null; + Assert.AreEqual(expected, unpacked); + } } } diff --git a/csharp/src/Google.Protobuf/FieldMaskTree.cs b/csharp/src/Google.Protobuf/FieldMaskTree.cs index 2297e7a11..63eb5f61c 100644 --- a/csharp/src/Google.Protobuf/FieldMaskTree.cs +++ b/csharp/src/Google.Protobuf/FieldMaskTree.cs @@ -333,15 +333,24 @@ namespace Google.Protobuf { if (sourceField != null) { - var sourceByteString = ((IMessage)sourceField).ToByteString(); - var destinationValue = (IMessage)field.Accessor.GetValue(destination); - if (destinationValue != null) + // Well-known wrapper types are represented as nullable primitive types, so we do not "merge" them. + // Instead, any non-null value just overwrites the previous value directly. + if (field.MessageType.IsWrapperType) { - destinationValue.MergeFrom(sourceByteString); + field.Accessor.SetValue(destination, sourceField); } else { - field.Accessor.SetValue(destination, field.MessageType.Parser.ParseFrom(sourceByteString)); + var sourceByteString = ((IMessage)sourceField).ToByteString(); + var destinationValue = (IMessage)field.Accessor.GetValue(destination); + if (destinationValue != null) + { + destinationValue.MergeFrom(sourceByteString); + } + else + { + field.Accessor.SetValue(destination, field.MessageType.Parser.ParseFrom(sourceByteString)); + } } } } diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index 79af7f3d8..40986da94 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -4,7 +4,7 @@ C# runtime library for Protocol Buffers - Google's data interchange format. Copyright 2015, Google Inc. Google Protocol Buffers - 3.20.1 + 3.21.0-rc1 7.2 Google Inc. diff --git a/csharp/src/Google.Protobuf/Reflection/Descriptor.cs b/csharp/src/Google.Protobuf/Reflection/Descriptor.cs index 5967a4fef..1cb43c0ea 100644 --- a/csharp/src/Google.Protobuf/Reflection/Descriptor.cs +++ b/csharp/src/Google.Protobuf/Reflection/Descriptor.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/descriptor.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs index 13066da3f..d0ce71b1d 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Any.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/any.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/AnyPartial.cs b/csharp/src/Google.Protobuf/WellKnownTypes/AnyPartial.cs index 5e7b6d5a1..cebbcd2fe 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/AnyPartial.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/AnyPartial.cs @@ -120,6 +120,26 @@ namespace Google.Protobuf.WellKnownTypes return true; } + /// + /// Attempts to unpack the content of this Any message into one of the message types + /// in the given type registry, based on the type URL. + /// + /// The type registry to consult for messages. + /// The unpacked message, or null if no matching message was found. + public IMessage Unpack(TypeRegistry registry) + { + string typeName = GetTypeName(TypeUrl); + MessageDescriptor descriptor = registry.Find(typeName); + if (descriptor == null) + { + return null; + } + + var message = descriptor.Parser.CreateTemplate(); + message.MergeFrom(Value); + return message; + } + /// /// Packs the specified message into an Any message using a type URL prefix of "type.googleapis.com". /// diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Api.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Api.cs index 58a658e07..c524031e3 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Api.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Api.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/api.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Duration.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Duration.cs index b46f4d293..576ef2e60 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Duration.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Duration.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/duration.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Empty.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Empty.cs index 08f4a849d..43d2b7490 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Empty.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Empty.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/empty.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs b/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs index e90c3d571..19e0e6e41 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/field_mask.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/SourceContext.cs b/csharp/src/Google.Protobuf/WellKnownTypes/SourceContext.cs index 58c23cea1..6edd4f139 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/SourceContext.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/SourceContext.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/source_context.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs index 123d80ac9..8c1eec53b 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Struct.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/struct.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Timestamp.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Timestamp.cs index e981dc2a0..e6d57afd5 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Timestamp.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Timestamp.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/timestamp.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Type.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Type.cs index 120f31a90..3088e385d 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Type.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Type.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/type.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/Wrappers.cs b/csharp/src/Google.Protobuf/WellKnownTypes/Wrappers.cs index 29c04b36b..307a446f0 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/Wrappers.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/Wrappers.cs @@ -2,7 +2,7 @@ // Generated by the protocol buffer compiler. DO NOT EDIT! // source: google/protobuf/wrappers.proto // -#pragma warning disable 1591, 0612, 3021 +#pragma warning disable 1591, 0612, 3021, 8981 #region Designer generated code using pb = global::Google.Protobuf; diff --git a/docs/options.md b/docs/options.md index b48928841..bda41c211 100644 --- a/docs/options.md +++ b/docs/options.md @@ -320,3 +320,7 @@ with info about your project (name and website) so we can add an entry for you. 1. Protoc-gen-authz * Website: https://github.com/Neakxs/protoc-gen-authz * Extension: 1145 + +1. Protonium + * Website: https://github.com/zyp/protonium + * Extension: 1146 diff --git a/java/README.md b/java/README.md index c5c0bfab5..41cbc869e 100644 --- a/java/README.md +++ b/java/README.md @@ -23,7 +23,7 @@ If you are using Maven, use the following: com.google.protobuf protobuf-java - 3.20.1 + 3.21.0-rc-1 ``` @@ -37,7 +37,7 @@ protobuf-java-util package: com.google.protobuf protobuf-java-util - 3.20.1 + 3.21.0-rc-1 ``` @@ -45,7 +45,7 @@ protobuf-java-util package: If you are using Gradle, add the following to your `build.gradle` file's dependencies: ``` - implementation 'com.google.protobuf:protobuf-java:3.20.1' + implementation 'com.google.protobuf:protobuf-java:3.21.0-rc-1' ``` Again, be sure to check that the version number matches (or is newer than) the version number of protoc that you are using. diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 045dc285a..ce950a209 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-bom - 3.20.1 + 3.21.0-rc-1 pom Protocol Buffers [BOM] diff --git a/java/core/BUILD.bazel b/java/core/BUILD.bazel index e36a38e7f..5a5eed972 100644 --- a/java/core/BUILD.bazel +++ b/java/core/BUILD.bazel @@ -3,7 +3,8 @@ load("@rules_java//java:defs.bzl", "java_library", "java_lite_proto_library", "j load("@rules_jvm_external//:defs.bzl", "java_export") load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain", "proto_library") -load("//:internal.bzl", "conformance_test") +load("//conformance:defs.bzl", "conformance_test") +load("//:protobuf.bzl", "internal_gen_well_known_protos_java") load("//:protobuf_version.bzl", "PROTOBUF_JAVA_VERSION") load("//java/internal:testing.bzl", "junit_tests") @@ -102,11 +103,28 @@ LITE_SRCS = [ "src/main/java/com/google/protobuf/Writer.java", ] +internal_gen_well_known_protos_java( + name = "gen_well_known_protos_javalite", + javalite = True, + deps = [ + "//:any_proto", + "//:api_proto", + "//:duration_proto", + "//:empty_proto", + "//:field_mask_proto", + "//:source_context_proto", + "//:struct_proto", + "//:timestamp_proto", + "//:type_proto", + "//:wrappers_proto", + ], +) + # Should be used as `//java/lite`. java_library( name = "lite", srcs = LITE_SRCS + [ - "//:gen_well_known_protos_javalite", + ":gen_well_known_protos_javalite", ], visibility = [ "//java/lite:__pkg__", @@ -130,6 +148,24 @@ java_library( srcs = LITE_SRCS, ) +internal_gen_well_known_protos_java( + name = "gen_well_known_protos_java", + deps = [ + "//:any_proto", + "//:api_proto", + "//:compiler_plugin_proto", + "//:descriptor_proto", + "//:duration_proto", + "//:empty_proto", + "//:field_mask_proto", + "//:source_context_proto", + "//:struct_proto", + "//:timestamp_proto", + "//:type_proto", + "//:wrappers_proto", + ], +) + java_library( name = "core", srcs = glob( @@ -138,7 +174,7 @@ java_library( ], exclude = LITE_SRCS, ) + [ - "//:gen_well_known_protos_java", + ":gen_well_known_protos_java", ], visibility = ["//visibility:public"], exports = [ @@ -155,7 +191,8 @@ java_export( maven_coordinates = "com.google.protobuf:protobuf-java:%s" % PROTOBUF_JAVA_VERSION, pom_template = "pom_template.xml", resources = [ - "//:well_known_protos", + "//src/google/protobuf:descriptor_proto_srcs", + "//:well_known_type_protos", ], tags = ["manual"], runtime_deps = [":core"], @@ -205,8 +242,9 @@ proto_library( deps = [ "//:any_proto", "//:descriptor_proto", - "//:generic_test_protos", + "//:lite_test_protos", "//:wrappers_proto", + "//src/google/protobuf:generic_test_protos", ], ) @@ -215,7 +253,15 @@ java_proto_library( visibility = [ "//java:__subpackages__", ], - deps = ["//:generic_test_protos"], + deps = ["//src/google/protobuf:generic_test_protos"], +) + +java_proto_library( + name = "lite_test_protos_java_proto", + visibility = [ + "//java:__subpackages__", + ], + deps = ["//:lite_test_protos"], ) java_proto_library( @@ -234,6 +280,7 @@ java_library( ":core", ":generic_test_protos_java_proto", ":java_test_protos_java_proto", + ":lite_test_protos_java_proto", "@maven//:com_google_guava_guava", "@maven//:junit_junit", ], @@ -258,9 +305,9 @@ build_test( conformance_test( name = "conformance_test", - failure_list = "//:conformance/failure_list_java.txt", - testee = "//:conformance_java", - text_format_failure_list = "//:conformance/text_format_failure_list_java.txt", + failure_list = "//conformance:failure_list_java.txt", + testee = "//conformance:conformance_java", + text_format_failure_list = "//conformance:text_format_failure_list_java.txt", ) junit_tests( @@ -275,11 +322,12 @@ junit_tests( "src/test/java/com/google/protobuf/TestUtilLite.java", ], ), - data = ["//:testdata"], + data = ["//src/google/protobuf:testdata"], deps = [ ":core", ":generic_test_protos_java_proto", ":java_test_protos_java_proto", + ":lite_test_protos_java_proto", ":test_util", "@maven//:com_google_guava_guava", "@maven//:com_google_truth_truth", @@ -309,9 +357,19 @@ junit_tests( java_lite_proto_library( name = "generic_test_protos_java_proto_lite", visibility = [ + "//java/kotlin-lite:__pkg__", "//java/lite:__pkg__", ], - deps = ["//:generic_test_protos"], + deps = ["//src/google/protobuf:generic_test_protos"], +) + +java_lite_proto_library( + name = "lite_test_protos_java_proto_lite", + visibility = [ + "//java/kotlin-lite:__pkg__", + "//java/lite:__pkg__", + ], + deps = ["//:lite_test_protos"], ) java_lite_proto_library( @@ -346,6 +404,7 @@ java_library( ":generic_test_protos_java_proto_lite", ":java_test_protos_java_proto_lite", ":lite_runtime_only", + ":lite_test_protos_java_proto_lite", "@maven//:com_google_guava_guava", "@maven//:junit_junit", ], @@ -402,12 +461,13 @@ junit_tests( ["src/test/java/**/*.java"], exclude = LITE_TEST_EXCLUSIONS, ), - data = ["//:testdata"], + data = ["//src/google/protobuf:testdata"], test_prefix = "Lite", deps = [ ":generic_test_protos_java_proto_lite", ":java_test_protos_java_proto_lite", ":lite", + ":lite_test_protos_java_proto_lite", ":test_util_lite", "@maven//:com_google_truth_truth", "@maven//:junit_junit", diff --git a/java/core/pom.xml b/java/core/pom.xml index bc0726c35..8a33a507a 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.20.1 + 3.21.0-rc-1 protobuf-java diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index a36195bbb..07d771582 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -461,21 +461,20 @@ public final class Descriptors { } /** - * This method is to be called by generated code only. It is used to update the + * This method is to be called by generated code only. It updates the * FileDescriptorProto associated with the descriptor by parsing it again with the given * ExtensionRegistry. This is needed to recognize custom options. */ public static void internalUpdateFileDescriptor( - final FileDescriptor descriptor, final ExtensionRegistry registry) { + FileDescriptor descriptor, ExtensionRegistry registry) { ByteString bytes = descriptor.proto.toByteString(); - FileDescriptorProto proto; try { - proto = FileDescriptorProto.parseFrom(bytes, registry); + FileDescriptorProto proto = FileDescriptorProto.parseFrom(bytes, registry); + descriptor.setProto(proto); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException( "Failed to parse protocol buffer descriptor for generated code.", e); } - descriptor.setProto(proto); } /** diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java index 9db33456d..e212ab5fe 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -244,11 +244,16 @@ public abstract class GeneratedMessageLite< * *

For use by generated code only. */ - protected abstract Object dynamicMethod(MethodToInvoke method, Object arg0, Object arg1); + protected abstract Object dynamicMethod( + MethodToInvoke method, + Object arg0, + Object arg1); /** Same as {@link #dynamicMethod(MethodToInvoke, Object, Object)} with {@code null} padding. */ @CanIgnoreReturnValue - protected Object dynamicMethod(MethodToInvoke method, Object arg0) { + protected Object dynamicMethod( + MethodToInvoke method, + Object arg0) { return dynamicMethod(method, arg0, null); } @@ -1245,11 +1250,11 @@ public abstract class GeneratedMessageLite< } @SuppressWarnings("unchecked") - Object fromFieldSetType(final Object value) { + Object fromFieldSetType(Object value) { if (descriptor.isRepeated()) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { - final List result = new ArrayList<>(); - for (final Object element : (List) value) { + List result = new ArrayList<>(); + for (Object element : (List) value) { result.add(singularFromFieldSetType(element)); } return result; @@ -1261,7 +1266,7 @@ public abstract class GeneratedMessageLite< } } - Object singularFromFieldSetType(final Object value) { + Object singularFromFieldSetType(Object value) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { return descriptor.enumTypeMap.findValueByNumber((Integer) value); } else { @@ -1269,12 +1274,11 @@ public abstract class GeneratedMessageLite< } } - @SuppressWarnings("unchecked") - Object toFieldSetType(final Object value) { + Object toFieldSetType(Object value) { if (descriptor.isRepeated()) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { - final List result = new ArrayList<>(); - for (final Object element : (List) value) { + List result = new ArrayList<>(); + for (Object element : (List) value) { result.add(singularToFieldSetType(element)); } return result; @@ -1286,7 +1290,7 @@ public abstract class GeneratedMessageLite< } } - Object singularToFieldSetType(final Object value) { + Object singularToFieldSetType(Object value) { if (descriptor.getLiteJavaType() == WireFormat.JavaType.ENUM) { return ((Internal.EnumLite) value).getNumber(); } else { diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index 7317f49ff..7fba30a04 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -116,8 +116,8 @@ public final class TextFormat { /** * Generates a human readable form of this message, useful for debugging and other purposes, with - * no newline characters. This is just a trivial wrapper around - * {@link TextFormat.Printer#shortDebugString(MessageOrBuilder)}. + * no newline characters. This is just a trivial wrapper around {@link + * TextFormat.Printer#shortDebugString(MessageOrBuilder)}. */ public static String shortDebugString(final MessageOrBuilder message) { return printer().shortDebugString(message); @@ -459,9 +459,7 @@ public final class TextFormat { } } - /** - * An adapter class that can take a {@link MapEntry} and returns its key and entry. - */ + /** An adapter class that can take a {@link MapEntry} and returns its key and entry. */ private static class MapEntryAdapter implements Comparable { private Object entry; @@ -953,6 +951,7 @@ public final class TextFormat { * the next token is parsed. */ private boolean containsSilentMarkerAfterCurrentToken = false; + private boolean containsSilentMarkerAfterPrevToken = false; /** Construct a tokenizer that parses tokens from the given text. */ @@ -1378,7 +1377,6 @@ public final class TextFormat { private ParseException floatParseException(final NumberFormatException e) { return parseException("Couldn't parse number: " + e.getMessage()); } - } /** Thrown when parsing an invalid text format message. */ @@ -1551,7 +1549,7 @@ public final class TextFormat { * the current token is part of the field value, so the silent marker is indicated by * containsSilentMarkerAfterPrevToken. */ - private void detectSilentMarker(Tokenizer tokenizer) { + private void detectSilentMarker(Tokenizer tokenizer, String fieldName) { } /** @@ -1628,8 +1626,8 @@ public final class TextFormat { * unknown field is encountered. If this is set, the parser will only log a warning. Allow * unknown fields will also allow unknown extensions. * - *

Use of this parameter is discouraged which may hide some errors (e.g. - * spelling error on field name). + *

Use of this parameter is discouraged which may hide some errors (e.g. spelling error on + * field name). */ public Builder setAllowUnknownFields(boolean allowUnknownFields) { this.allowUnknownFields = allowUnknownFields; @@ -1637,10 +1635,9 @@ public final class TextFormat { } /** - * Set whether this parser will allow unknown extensions. By default, an - * exception is thrown if unknown extension is encountered. If this is set true, - * the parser will only log a warning. Allow unknown extensions does not mean - * allow normal unknown fields. + * Set whether this parser will allow unknown extensions. By default, an exception is thrown + * if unknown extension is encountered. If this is set true, the parser will only log a + * warning. Allow unknown extensions does not mean allow normal unknown fields. */ public Builder setAllowUnknownExtensions(boolean allowUnknownExtensions) { this.allowUnknownExtensions = allowUnknownExtensions; @@ -1725,7 +1722,8 @@ public final class TextFormat { static final class UnknownField { static enum Type { - FIELD, EXTENSION; + FIELD, + EXTENSION; } final String message; @@ -1786,7 +1784,6 @@ public final class TextFormat { throws ParseException { final Tokenizer tokenizer = new Tokenizer(input); MessageReflection.BuilderAdapter target = new MessageReflection.BuilderAdapter(builder); - List unknownFields = new ArrayList(); while (!tokenizer.atEnd()) { @@ -1803,12 +1800,7 @@ public final class TextFormat { final MessageReflection.MergeTarget target, List unknownFields) throws ParseException { - mergeField( - tokenizer, - extensionRegistry, - target, - parseInfoTreeBuilder, - unknownFields); + mergeField(tokenizer, extensionRegistry, target, parseInfoTreeBuilder, unknownFields); } /** Parse a single field from {@code tokenizer} and merge it into {@code target}. */ @@ -1820,26 +1812,28 @@ public final class TextFormat { List unknownFields) throws ParseException { FieldDescriptor field = null; + String name; int startLine = tokenizer.getLine(); int startColumn = tokenizer.getColumn(); final Descriptor type = target.getDescriptorForType(); ExtensionRegistry.ExtensionInfo extension = null; if ("google.protobuf.Any".equals(type.getFullName()) && tokenizer.tryConsume("[")) { - mergeAnyFieldValue(tokenizer, extensionRegistry, target, parseTreeBuilder, unknownFields, - type); + mergeAnyFieldValue( + tokenizer, extensionRegistry, target, parseTreeBuilder, unknownFields, type); return; } if (tokenizer.tryConsume("[")) { // An extension. - final StringBuilder name = new StringBuilder(tokenizer.consumeIdentifier()); + StringBuilder nameBuilder = new StringBuilder(tokenizer.consumeIdentifier()); while (tokenizer.tryConsume(".")) { - name.append('.'); - name.append(tokenizer.consumeIdentifier()); + nameBuilder.append('.'); + nameBuilder.append(tokenizer.consumeIdentifier()); } + name = nameBuilder.toString(); - extension = target.findExtensionByName(extensionRegistry, name.toString()); + extension = target.findExtensionByName(extensionRegistry, name); if (extension == null) { String message = @@ -1866,7 +1860,7 @@ public final class TextFormat { tokenizer.consume("]"); } else { - final String name = tokenizer.consumeIdentifier(); + name = tokenizer.consumeIdentifier(); field = type.findFieldByName(name); // Group names are expected to be capitalized as they appear in the @@ -1890,13 +1884,14 @@ public final class TextFormat { } if (field == null) { - String message = (tokenizer.getPreviousLine() + 1) - + ":" - + (tokenizer.getPreviousColumn() + 1) - + ":\t" - + type.getFullName() - + "." - + name; + String message = + (tokenizer.getPreviousLine() + 1) + + ":" + + (tokenizer.getPreviousColumn() + 1) + + ":\t" + + type.getFullName() + + "." + + name; unknownFields.add(new UnknownField(message, UnknownField.Type.FIELD)); } } @@ -1909,7 +1904,7 @@ public final class TextFormat { // start with "{" or "<" which indicates the beginning of a message body. // If there is no ":" or there is a "{" or "<" after ":", this field has // to be a message or the input is ill-formed. - detectSilentMarker(tokenizer); + detectSilentMarker(tokenizer, name); if (tokenizer.tryConsume(":") && !tokenizer.lookingAt("{") && !tokenizer.lookingAt("<")) { skipFieldValue(tokenizer); } else { @@ -1920,7 +1915,7 @@ public final class TextFormat { // Handle potential ':'. if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { - detectSilentMarker(tokenizer); + detectSilentMarker(tokenizer, field.getFullName()); tokenizer.tryConsume(":"); // optional if (parseTreeBuilder != null) { TextFormatParseInfoTree.Builder childParseTreeBuilder = @@ -1944,7 +1939,7 @@ public final class TextFormat { unknownFields); } } else { - detectSilentMarker(tokenizer); + detectSilentMarker(tokenizer, field.getFullName()); tokenizer.consume(":"); // required consumeFieldValues( tokenizer, @@ -1967,6 +1962,29 @@ public final class TextFormat { } } + private String consumeFullTypeName(Tokenizer tokenizer) throws ParseException { + // If there is not a leading `[`, this is just a type name. + if (!tokenizer.tryConsume("[")) { + return tokenizer.consumeIdentifier(); + } + + // Otherwise, this is an extension or google.protobuf.Any type URL: we consume proto path + // elements until we've addressed the type. + String name = tokenizer.consumeIdentifier(); + while (tokenizer.tryConsume(".")) { + name += "." + tokenizer.consumeIdentifier(); + } + if (tokenizer.tryConsume("/")) { + name += "/" + tokenizer.consumeIdentifier(); + while (tokenizer.tryConsume(".")) { + name += "." + tokenizer.consumeIdentifier(); + } + } + tokenizer.consume("]"); + + return name; + } + /** * Parse a one or more field values from {@code tokenizer} and merge it into {@code builder}. */ @@ -2058,8 +2076,13 @@ public final class TextFormat { // (java_proto_library for any_java_proto depends on the protobuf_impl). Message anyBuilder = DynamicMessage.getDefaultInstance(field.getMessageType()); MessageReflection.MergeTarget anyField = target.newMergeTargetForField(field, anyBuilder); - mergeAnyFieldValue(tokenizer, extensionRegistry, anyField, parseTreeBuilder, - unknownFields, field.getMessageType()); + mergeAnyFieldValue( + tokenizer, + extensionRegistry, + anyField, + parseTreeBuilder, + unknownFields, + field.getMessageType()); value = anyField.finish(); tokenizer.consume(endToken); } else { @@ -2206,7 +2229,7 @@ public final class TextFormat { throw tokenizer.parseExceptionPreviousToken("Expected a valid type URL."); } } - detectSilentMarker(tokenizer); + detectSilentMarker(tokenizer, typeUrlBuilder.toString()); tokenizer.tryConsume(":"); final String anyEndToken; if (tokenizer.tryConsume("<")) { @@ -2244,15 +2267,7 @@ public final class TextFormat { /** Skips the next field including the field's name and value. */ private void skipField(Tokenizer tokenizer) throws ParseException { - if (tokenizer.tryConsume("[")) { - // Extension name. - do { - tokenizer.consumeIdentifier(); - } while (tokenizer.tryConsume(".")); - tokenizer.consume("]"); - } else { - tokenizer.consumeIdentifier(); - } + String name = consumeFullTypeName(tokenizer); // Try to guess the type of this field. // If this field is not a message, there should be a ":" between the @@ -2260,7 +2275,7 @@ public final class TextFormat { // start with "{" or "<" which indicates the beginning of a message body. // If there is no ":" or there is a "{" or "<" after ":", this field has // to be a message or the input is ill-formed. - detectSilentMarker(tokenizer); + detectSilentMarker(tokenizer, name); if (tokenizer.tryConsume(":") && !tokenizer.lookingAt("<") && !tokenizer.lookingAt("{")) { skipFieldValue(tokenizer); } else { @@ -2469,9 +2484,10 @@ public final class TextFormat { } Character.UnicodeBlock unicodeBlock = Character.UnicodeBlock.of(codepoint); if (unicodeBlock != null - && (unicodeBlock.equals(Character.UnicodeBlock.LOW_SURROGATES) - || unicodeBlock.equals(Character.UnicodeBlock.HIGH_SURROGATES) - || unicodeBlock.equals(Character.UnicodeBlock.HIGH_PRIVATE_USE_SURROGATES))) { + && (unicodeBlock.equals(Character.UnicodeBlock.LOW_SURROGATES) + || unicodeBlock.equals(Character.UnicodeBlock.HIGH_SURROGATES) + || unicodeBlock.equals( + Character.UnicodeBlock.HIGH_PRIVATE_USE_SURROGATES))) { throw new InvalidEscapeSequenceException( "Invalid escape sequence: '\\U" + input.substring(i, i + 8).toStringUtf8() diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java index cde177683..1689c72e9 100644 --- a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java +++ b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java @@ -67,9 +67,7 @@ import org.junit.function.ThrowingRunnable; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** - * Test case for {@link TextFormat}. - */ +/** Test case for {@link TextFormat}. */ @RunWith(JUnit4.class) public class TextFormatTest { @@ -1450,6 +1448,18 @@ public class TextFormatTest { + "unknown_field3: 3\n"); } + @Test + public void testParseUnknownExtensionWithAnyMessage() throws Exception { + assertParseSuccessWithUnknownExtensions( + "[unknown_extension]: { " + + " any_value { " + + " [type.googleapis.com/protobuf_unittest.OneString] { " + + " data: 123 " + + " } " + + " } " + + "}"); + } + // See additional coverage in testOneofOverwriteForbidden and testMapOverwriteForbidden. @Test public void testParseNonRepeatedFields() throws Exception { diff --git a/java/kotlin-lite/BUILD.bazel b/java/kotlin-lite/BUILD.bazel index 09f0decff..39f8b33ee 100644 --- a/java/kotlin-lite/BUILD.bazel +++ b/java/kotlin-lite/BUILD.bazel @@ -2,8 +2,8 @@ load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") load("@rules_java//java:defs.bzl", "java_lite_proto_library") load("@rules_jvm_external//:kt_defs.bzl", "kt_jvm_export") load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") -load("//:protobuf_version.bzl", "PROTOBUF_JAVA_VERSION") load("//:protobuf.bzl", "internal_gen_kt_protos") +load("//:protobuf_version.bzl", "PROTOBUF_JAVA_VERSION") java_lite_proto_library( name = "example_extensible_message_java_proto_lite", @@ -16,10 +16,27 @@ kt_jvm_library( deps = ["//java/lite"], ) +internal_gen_kt_protos( + name = "gen_well_known_protos_kotlinlite", + lite = True, + deps = [ + "//:any_proto", + "//:api_proto", + "//:duration_proto", + "//:empty_proto", + "//:field_mask_proto", + "//:source_context_proto", + "//:struct_proto", + "//:timestamp_proto", + "//:type_proto", + "//:wrappers_proto", + ], +) + kt_jvm_library( name = "well_known_protos_kotlin_lite", srcs = [ - "//:gen_well_known_protos_kotlinlite", + ":gen_well_known_protos_kotlinlite", ], deps = [ "//java/kotlin:only_for_use_in_proto_generated_code_its_generator_and_tests", @@ -36,7 +53,7 @@ kt_jvm_export( ], maven_coordinates = "com.google.protobuf:protobuf-kotlin-lite:%s" % PROTOBUF_JAVA_VERSION, pom_template = "//java/kotlin-lite:pom_template.xml", - resources = ["//:well_known_protos"], + resources = ["//:well_known_type_protos"], tags = ["manual"], runtime_deps = [ ":lite_extensions", @@ -120,32 +137,46 @@ internal_gen_kt_protos( deps = ["//java/kotlin:multiple_files_proto3"], ) +internal_gen_kt_protos( + name = "gen_kotlin_unittest_lite", + lite = True, + deps = ["//src/google/protobuf:lite_test_protos"], +) + kt_jvm_library( name = "kotlin_unittest_lite", srcs = [ ":gen_evil_names_proto2_lite", - "//:gen_kotlin_unittest_lite", + ":gen_kotlin_unittest_lite", ], deps = [ ":evil_names_proto2_java_proto_lite", - "//:java_lite_test_protos", + "//java/core:generic_test_protos_java_proto_lite", + "//java/core:lite_test_protos_java_proto_lite", "//java/kotlin:only_for_use_in_proto_generated_code_its_generator_and_tests", "//java/kotlin:shared_runtime", "//java/lite", ], ) +internal_gen_kt_protos( + name = "gen_kotlin_proto3_unittest_lite", + lite = True, + deps = ["//src/google/protobuf:kt_proto3_unittest_protos"], +) + kt_jvm_library( name = "kotlin_proto3_unittest_lite", srcs = [ ":gen_evil_names_proto3_lite", ":gen_kotlin_proto3_java_multiple_files_lite", - "//:gen_kotlin_proto3_unittest_lite", + ":gen_kotlin_proto3_unittest_lite", ], deps = [ ":evil_names_proto3_java_proto_lite", ":multiple_files_proto3_java_proto_lite", - "//:java_lite_test_protos", + "//java/core:generic_test_protos_java_proto_lite", + "//java/core:lite_test_protos_java_proto_lite", "//java/kotlin:only_for_use_in_proto_generated_code_its_generator_and_tests", "//java/kotlin:shared_runtime", "//java/lite", diff --git a/java/kotlin-lite/pom.xml b/java/kotlin-lite/pom.xml index d0ef96d46..a4737d829 100644 --- a/java/kotlin-lite/pom.xml +++ b/java/kotlin-lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.20.1 + 3.21.0-rc-1 protobuf-kotlin-lite diff --git a/java/kotlin/BUILD.bazel b/java/kotlin/BUILD.bazel index 859b86ba3..9814c7a0d 100644 --- a/java/kotlin/BUILD.bazel +++ b/java/kotlin/BUILD.bazel @@ -58,7 +58,10 @@ kt_jvm_export( ], maven_coordinates = "com.google.protobuf:protobuf-kotlin:%s" % PROTOBUF_JAVA_VERSION, pom_template = "//java/kotlin:pom_template.xml", - resources = ["//:well_known_protos"], + resources = [ + "//src/google/protobuf:descriptor_proto_srcs", + "//:well_known_type_protos", + ], tags = ["manual"], runtime_deps = [ ":bytestring_lib", @@ -218,36 +221,49 @@ internal_gen_kt_protos( deps = [":multiple_files_proto3"], ) +internal_gen_kt_protos( + name = "gen_kotlin_unittest", + visibility = ["//java:__subpackages__"], + deps = ["//src/google/protobuf:kt_unittest_protos"], +) + kt_jvm_library( name = "kotlin_unittest", srcs = [ ":gen_evil_names_proto2", - "//:gen_kotlin_unittest", + ":gen_kotlin_unittest", ], deps = [ ":evil_names_proto2_java_proto", ":only_for_use_in_proto_generated_code_its_generator_and_tests", ":shared_runtime", ":well_known_protos_kotlin", - "//:java_test_protos", "//java/core", + "//java/core:generic_test_protos_java_proto", + "//java/core:lite_test_protos_java_proto", ], ) +internal_gen_kt_protos( + name = "gen_kotlin_proto3_unittest", + deps = ["//src/google/protobuf:kt_proto3_unittest_protos"], +) + kt_jvm_library( name = "kotlin_proto3_unittest", srcs = [ ":gen_evil_names_proto3", ":gen_kotlin_proto3_java_multiple_files", - "//:gen_kotlin_proto3_unittest", + ":gen_kotlin_proto3_unittest", ], deps = [ ":evil_names_proto3_java_proto", ":multiple_files_proto3_java_proto", ":only_for_use_in_proto_generated_code_its_generator_and_tests", ":shared_runtime", - "//:java_test_protos", "//java/core", + "//java/core:generic_test_protos_java_proto", + "//java/core:lite_test_protos_java_proto", ], ) @@ -285,10 +301,31 @@ java_test( runtime_deps = [":proto3_test_library"], ) +internal_gen_kt_protos( + name = "gen_well_known_protos_kotlin", + visibility = [ + "//java:__subpackages__", + ], + deps = [ + "//:any_proto", + "//:api_proto", + "//:compiler_plugin_proto", + "//:descriptor_proto", + "//:duration_proto", + "//:empty_proto", + "//:field_mask_proto", + "//:source_context_proto", + "//:struct_proto", + "//:timestamp_proto", + "//:type_proto", + "//:wrappers_proto", + ], +) + kt_jvm_library( name = "well_known_protos_kotlin", srcs = [ - "//:gen_well_known_protos_kotlin", + ":gen_well_known_protos_kotlin", ], deps = [ ":only_for_use_in_proto_generated_code_its_generator_and_tests", diff --git a/java/kotlin/pom.xml b/java/kotlin/pom.xml index 591c4ae38..a3f66409b 100644 --- a/java/kotlin/pom.xml +++ b/java/kotlin/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.20.1 + 3.21.0-rc-1 protobuf-kotlin diff --git a/java/lite.md b/java/lite.md index 0620a66fe..fd6c163c1 100644 --- a/java/lite.md +++ b/java/lite.md @@ -29,7 +29,7 @@ protobuf Java Lite runtime. If you are using Maven, include the following: com.google.protobuf protobuf-javalite - 3.20.1 + 3.21.0-rc-1 ``` diff --git a/java/lite/BUILD.bazel b/java/lite/BUILD.bazel index b492cdda7..f84b5c514 100644 --- a/java/lite/BUILD.bazel +++ b/java/lite/BUILD.bazel @@ -1,7 +1,7 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_pkg//:mappings.bzl", "pkg_filegroup", "pkg_files", "strip_prefix") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -load("//:internal.bzl", "conformance_test") +load("//conformance:defs.bzl", "conformance_test") load("//java/internal:testing.bzl", "junit_tests") exports_files( @@ -59,9 +59,9 @@ build_test( conformance_test( name = "conformance_test", - failure_list = "//:conformance/failure_list_java_lite.txt", - testee = "//:conformance_java_lite", - text_format_failure_list = "//:conformance/text_format_failure_list_java_lite.txt", + failure_list = "//conformance:failure_list_java_lite.txt", + testee = "//conformance:conformance_java_lite", + text_format_failure_list = "//conformance:text_format_failure_list_java_lite.txt", ) junit_tests( @@ -72,6 +72,7 @@ junit_tests( ":lite", "//java/core:generic_test_protos_java_proto_lite", "//java/core:java_test_protos_java_proto_lite", + "//java/core:lite_test_protos_java_proto_lite", "//java/core:test_util_lite", "@maven//:com_google_truth_truth", "@maven//:junit_junit", diff --git a/java/lite/pom.xml b/java/lite/pom.xml index 8055508e4..d62c1b375 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.20.1 + 3.21.0-rc-1 protobuf-javalite diff --git a/java/pom.xml b/java/pom.xml index 3c44917e6..a1d6ab06d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.20.1 + 3.21.0-rc-1 pom Protocol Buffers [Parent] diff --git a/java/util/pom.xml b/java/util/pom.xml index cd5f96e26..f20e1e5c5 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -4,7 +4,7 @@ com.google.protobuf protobuf-parent - 3.20.1 + 3.21.0-rc-1 protobuf-java-util diff --git a/kokoro/caplog.sh b/kokoro/common/caplog.sh similarity index 98% rename from kokoro/caplog.sh rename to kokoro/common/caplog.sh index fe81949ea..33ee1abdf 100644 --- a/kokoro/caplog.sh +++ b/kokoro/common/caplog.sh @@ -19,7 +19,7 @@ # # Use the provided functions below as build/test fixtures, e.g.: # -# source kokoro/capture_logs.sh +# source kokoro/common/capture_logs.sh # caplog build/step1 # caplog tests/step2 # diff --git a/kokoro/common/check_missing_dist_files.sh b/kokoro/common/check_missing_dist_files.sh new file mode 100755 index 000000000..db4d92fc1 --- /dev/null +++ b/kokoro/common/check_missing_dist_files.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Script to compare a distribution archive for expected files based on git. +# +# Usage: +# check_missing_dist_files.sh path/to/dist_archive.tar.gz + +set -eux +set -o pipefail + +# By default, look for a git repo based on this script's path. +: ${SOURCE_DIR:=$(cd $(dirname $0)/../.. ; pwd)} + +# Use a temporary directory for intermediate files. +# Note that pipelines below use subshells to avoid multiple trap executions. +_workdir=$(mktemp -d) +function cleanup_workdir() { rm -r ${_workdir}; } +trap cleanup_workdir EXIT + +# List all the files in the archive. +( + tar -atf $1 | \ + cut -d/ -f2- | \ + sort +) > ${_workdir}/archive.lst + +# List all files in the git repo that should be in the archive. +( + git -C ${SOURCE_DIR} ls-files | \ + grep "^\(java\|python\|objectivec\|csharp\|ruby\|php\|cmake\|examples\|src/google/protobuf/.*\.proto\)" |\ + grep -v ".gitignore" | \ + grep -v "java/lite/proguard.pgcfg" | \ + grep -v "python/compatibility_tests" | \ + grep -v "python/docs" | \ + grep -v "python/.repo-metadata.json" | \ + grep -v "python/protobuf_distutils" | \ + grep -v "csharp/compatibility_tests" | \ + sort +) > ${_workdir}/expected.lst + +# Check for missing files. +MISSING_FILES=( $(cd ${_workdir} && comm -13 archive.lst expected.lst) ) +if (( ${#MISSING_FILES[@]} == 0 )); then + exit 0 +fi + +( + set +x + echo -e "\n\nMissing files from archive:" + for (( i=0 ; i < ${#MISSING_FILES[@]} ; i++ )); do + echo " ${MISSING_FILES[i]}" + done + echo -e "\nAdd them to the 'pkg_files' rule in corresponding BUILD.bazel.\n" +) >&2 +exit 1 diff --git a/kokoro/common/cmake.sh b/kokoro/common/cmake.sh new file mode 100755 index 000000000..99be3bef2 --- /dev/null +++ b/kokoro/common/cmake.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# +# Build tests under CMake. +# +# This script is used from macos and linux builds. It runs cmake and ctest in +# the current directory. Any additional setup should be done before running this +# script. +# +# This script uses `caplog` to save logfiles. See caplog.sh for details. + +set -eu -o pipefail +: ${SCRIPT_ROOT:=$(cd $(dirname $0)/../..; pwd)} + +################################################################################ +# If you are using this script to run tests, you can set some environment +# variables to control behavior: +# +# By default, find the sources based on this script's path. +: ${SOURCE_DIR:=${SCRIPT_ROOT}} +# +# By default, put outputs under /cmake/build. +: ${BUILD_DIR:=${SOURCE_DIR}/cmake/build} +# +# CMAKE_BUILD_TYPE is supported in cmake 3.22+. If set, we pass the value of this +# variable explicitly for compatibility with older versions of cmake. See: +# https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html +# (N.B.: not to be confused with CMAKE_CONFIG_TYPE.) +if [[ -n ${CMAKE_BUILD_TYPE:-} ]]; then + CMAKE_BUILD_TYPE_FLAG="-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}" +else + CMAKE_BUILD_TYPE_FLAG= +fi +# +# For several other CMake options, see docs here: +# https://cmake.org/cmake/help/latest/manual/cmake-env-variables.7.html +# +# Some variables you may want to override (see cmake docs for details): +# CMAKE_BUILD_PARALLEL_LEVEL +# CMAKE_CONFIG_TYPE (N.B.: not to be confused with CMAKE_BUILD_TYPE) +# CMAKE_GENERATOR +# CTEST_PARALLEL_LEVEL +################################################################################ + +echo "Building using..." +echo " Sources: ${SOURCE_DIR}" +echo " Build output: ${BUILD_DIR}" +if [[ ${SOURCE_DIR} != ${SCRIPT_ROOT} ]]; then + echo " Build scripts: ${SCRIPT_ROOT}" +fi +set -x +source ${SCRIPT_ROOT}/kokoro/common/caplog.sh + +# +# Configure under $BUILD_DIR: +# +mkdir -p "${BUILD_DIR}" + +( + cd "${BUILD_DIR}" + caplog 01_configure \ + cmake -S "${SOURCE_DIR}" \ + ${CMAKE_BUILD_TYPE_FLAG} \ + ${CAPLOG_CMAKE_ARGS:-} +) +if [[ -n ${CAPLOG_DIR:-} ]]; then + # Save configuration logs. + mkdir -p "${CAPLOG_DIR}/CMakeFiles" + cp "${BUILD_DIR}"/CMakeFiles/CMake*.log "${CAPLOG_DIR}/CMakeFiles" +fi + +# +# Build: +# +caplog 02_build \ + cmake --build "${BUILD_DIR}" + +# +# Run tests +# +( + cd "${BUILD_DIR}" + caplog 03_combined_testlog \ + ctest ${CAPLOG_CTEST_ARGS:-} +) diff --git a/kokoro/docs/publish-python.sh b/kokoro/docs/publish-python.sh index e6caf2df9..684afa6e5 100755 --- a/kokoro/docs/publish-python.sh +++ b/kokoro/docs/publish-python.sh @@ -14,8 +14,8 @@ sudo apt-get -y install software-properties-common sudo add-apt-repository universe sudo apt-get update sudo apt-get -y install unzip -wget https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip -unzip protoc-3.15.8-linux-x86_64.zip bin/protoc +wget https://github.com/protocolbuffers/protobuf/releases/download/v21.0-rc1/protoc-21.0-rc-1-linux-x86_64.zip +unzip protoc-21.0-rc-1-linux-x86_64.zip bin/protoc mv bin/protoc ../src/protoc python3 -m venv venv source venv/bin/activate @@ -30,7 +30,7 @@ make html cd .. deactivate -python3 -m pip install protobuf==3.15.8 gcp-docuploader +python3 -m pip install protobuf==4.21.0rc1 gcp-docuploader # install a json parser sudo apt-get -y install jq diff --git a/kokoro/linux/bazel/build.sh b/kokoro/linux/bazel/build.sh index d0d4b3ef4..fd0f51f82 100755 --- a/kokoro/linux/bazel/build.sh +++ b/kokoro/linux/bazel/build.sh @@ -7,16 +7,6 @@ set -ex use_bazel.sh 4.0.0 bazel version -# Print bazel testlogs to stdout when tests failed. -function print_test_logs { - # TODO(yannic): Only print logs of failing tests. - testlogs_dir=$(bazel info bazel-testlogs) - testlogs=$(find "${testlogs_dir}" -name "*.log") - for log in $testlogs; do - cat "${log}" - done -} - # Change to repo root cd $(dirname $0)/../../.. @@ -25,19 +15,13 @@ git submodule update --init --recursive # Disabled for now, re-enable if appropriate. # //:build_files_updated_unittest \ -trap print_test_logs EXIT -bazel test -k --copt=-Werror --host_copt=-Werror \ +bazel test \ + -k --copt=-Werror --host_copt=-Werror --test_output=errors \ + //build_defs:all \ //java:tests \ - //:protoc \ - //:protobuf \ + //src/... \ //:protobuf_python \ - //:protobuf_test \ - @com_google_protobuf//:cc_proto_blacklist_test -trap - EXIT - -pushd examples -bazel build //... -popd + @com_google_protobuf_examples//... # Verify that we can build successfully from generated tar files. ./autogen.sh && ./configure && make -j$(nproc) dist diff --git a/kokoro/linux/bazel_distcheck/build.sh b/kokoro/linux/bazel_distcheck/build.sh new file mode 100755 index 000000000..b122761bd --- /dev/null +++ b/kokoro/linux/bazel_distcheck/build.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# +# Build file to set up and run tests using bazel-build dist archive +# +# Note that the builds use WORKSPACE to fetch external sources, not +# git submodules. + +set -eux + +BUILD_ONLY_TARGETS=( + //pkg:all + //:protoc + //:protobuf + //:protobuf_python +) + +TEST_TARGETS=( + //build_defs:all + //conformance:all + //java:tests + //python:all + //src/... + @com_google_protobuf_examples//... +) + +CONTAINER_NAME=gcr.io/protobuf-build/bazel/linux +CONTAINER_VERSION=5.1.1-e41ccfa1648716433276ebe077c665796550fcbb + +use_bazel.sh 5.0.0 || true +bazel version + +# Change to repo root +cd $(dirname $0)/../../.. + +# Construct temp directory for running the dist build. +# If you want to run locally and keep the build dir, create a directory +# and pass it in the DIST_WORK_ROOT env var. +if [[ -z ${DIST_WORK_ROOT:-} ]]; then + : ${DIST_WORK_ROOT:=$(mktemp -d)} + function dist_cleanup() { + rm -rf ${DIST_WORK_ROOT} + } + trap dist_cleanup EXIT +fi + +# Let Bazel share the distdir. +TMP_DISTDIR=${DIST_WORK_ROOT}/bazel-distdir +mkdir -p ${TMP_DISTDIR} + +# Build distribution archive +date +bazel fetch --distdir=${TMP_DISTDIR} //pkg:dist_all_tar +bazel build --distdir=${TMP_DISTDIR} //pkg:dist_all_tar +DIST_ARCHIVE=$(readlink $(bazel info bazel-bin)/pkg/dist_all_tar.tar.gz) +bazel shutdown + +# The `pkg_tar` rule emits a symlink based on the rule name. The actual +# file is named with the current version. +date +echo "Resolved archive path: ${DIST_ARCHIVE}" + +# Extract the dist archive. +date +DIST_WORKSPACE=${DIST_WORK_ROOT}/protobuf +mkdir -p ${DIST_WORKSPACE} +tar -C ${DIST_WORKSPACE} --strip-components=1 -axf ${DIST_ARCHIVE} + +# Perform build steps in the extracted dist sources. + +cd ${DIST_WORKSPACE} +FAILED=false + +until docker pull gcr.io/protobuf-build/bazel/linux:${CONTAINER_VERSION}; do + sleep 10 +done + +date +docker run --rm \ + -v ${DIST_WORKSPACE}:/workspace \ + -v ${TMP_DISTDIR}:${TMP_DISTDIR} \ + ${CONTAINER_NAME}:${CONTAINER_VERSION} \ + test --distdir=${TMP_DISTDIR} --test_output=errors -k \ + "${BUILD_ONLY_TARGETS[@]}" "${TEST_TARGETS[@]}" || FAILED=true + +if ${FAILED}; then + echo FAILED + exit 1 +fi +echo PASS diff --git a/kokoro/linux/bazel_distcheck/continuous.cfg b/kokoro/linux/bazel_distcheck/continuous.cfg new file mode 100644 index 000000000..4ea8b2100 --- /dev/null +++ b/kokoro/linux/bazel_distcheck/continuous.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh" +timeout_mins: 15 diff --git a/kokoro/linux/bazel_distcheck/presubmit.cfg b/kokoro/linux/bazel_distcheck/presubmit.cfg new file mode 100644 index 000000000..4ea8b2100 --- /dev/null +++ b/kokoro/linux/bazel_distcheck/presubmit.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh" +timeout_mins: 15 diff --git a/kokoro/linux/cmake_distcheck/build.sh b/kokoro/linux/cmake_distcheck/build.sh new file mode 100755 index 000000000..5e957f8a5 --- /dev/null +++ b/kokoro/linux/cmake_distcheck/build.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Build file to set up and run tests based on distribution archive + +set -eux + +# Change to repo root +cd $(dirname $0)/../../.. + +# +# Update git submodules +# +git submodule update --init --recursive + +# +# Build distribution archive +# +# TODO: this should use Bazel-built dist archives. +date ; ./autogen.sh +date ; ./configure +date ; make dist +date + +DIST_ARCHIVE=( $(ls protobuf-*.tar.gz) ) +if (( ${#DIST_ARCHIVE[@]} != 1 )); then + echo >&2 "Distribution archive not found. ${#DIST_ARCHIVE[@]} matches:" + echo >&2 "${DIST_ARCHIVE[@]}" + exit 1 +fi + +# +# Check for all expected files +# +kokoro/common/check_missing_dist_files.sh ${DIST_ARCHIVE} + +# +# Extract to a temporary directory +# +if [[ -z ${DIST_WORK_ROOT:-} ]]; then + # If you want to preserve the extracted sources, set the DIST_WORK_ROOT + # environment variable to an existing directory that should be used. + DIST_WORK_ROOT=$(mktemp -d) + function cleanup_work_root() { + echo "Cleaning up temporary directory ${DIST_WORK_ROOT}..." + rm -rf ${DIST_WORK_ROOT} + } + trap cleanup_work_root EXIT +fi + +tar -C ${DIST_WORK_ROOT} --strip-components=1 -axf ${DIST_ARCHIVE} + +# +# Run tests using extracted sources +# +if SOURCE_DIR=${DIST_WORK_ROOT} \ + CMAKE_GENERATOR=Ninja \ + CTEST_PARALLEL_LEVEL=$(nproc) \ + kokoro/common/cmake.sh; then + # TODO: remove this conditional. + # The cmake build is expected to fail due to missing abseil sources. + echo "$0: Expected failure, but build passed." >&2 + echo "Please update $(basename $0) to remove failure expectation." >&2 + echo "FAIL" >&2 + exit 1 +fi +echo "PASS" diff --git a/kokoro/linux/cmake_distcheck/continuous.cfg b/kokoro/linux/cmake_distcheck/continuous.cfg new file mode 100644 index 000000000..6ef4c890b --- /dev/null +++ b/kokoro/linux/cmake_distcheck/continuous.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake_distcheck/build.sh" +timeout_mins: 1440 diff --git a/kokoro/linux/cmake_distcheck/presubmit.cfg b/kokoro/linux/cmake_distcheck/presubmit.cfg new file mode 100644 index 000000000..6ef4c890b --- /dev/null +++ b/kokoro/linux/cmake_distcheck/presubmit.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/cmake_distcheck/build.sh" +timeout_mins: 1440 diff --git a/kokoro/macos-next/cpp/build.sh b/kokoro/macos-next/cpp/build.sh index 490e99016..451c8508d 100755 --- a/kokoro/macos-next/cpp/build.sh +++ b/kokoro/macos-next/cpp/build.sh @@ -9,47 +9,21 @@ if [[ -h /tmpfs ]] && [[ ${PWD} == /tmpfs/src ]]; then cd /Volumes/BuildData/tmpfs/src fi -# These vars can be changed when running manually, e.g.: -# -# % BUILD_CONFIG=RelWithDebInfo path/to/build.sh +# Default environment variables used by cmake build: +: ${CMAKE_CONFIG_TYPE:=Debug} +export CMAKE_CONFIG_TYPE +: ${CTEST_PARALLEL_LEVEL:=4} +export CTEST_PARALLEL_LEVEL -# By default, build using Debug config. -: ${BUILD_CONFIG:=Debug} - -# By default, find the sources based on this script path. -: ${SOURCE_DIR:=$(cd $(dirname $0)/../../..; pwd)} - -# By default, put outputs under /cmake/build. -: ${BUILD_DIR:=${SOURCE_DIR}/cmake/build} - -source ${SOURCE_DIR}/kokoro/caplog.sh +# Run from the project root directory. +cd $(dirname $0)/../../.. # # Update submodules # -git -C "${SOURCE_DIR}" submodule update --init --recursive +git submodule update --init --recursive # -# Configure and build in a separate directory +# Run build # -mkdir -p "${BUILD_DIR}" - -caplog 01_configure \ - cmake -S "${SOURCE_DIR}" -B "${BUILD_DIR}" ${CAPLOG_CMAKE_ARGS:-} - -if [[ -n ${CAPLOG_DIR:-} ]]; then - mkdir -p "${CAPLOG_DIR}/CMakeFiles" - cp "${BUILD_DIR}"/CMakeFiles/CMake*.log "${CAPLOG_DIR}/CMakeFiles" -fi - -caplog 02_build \ - cmake --build "${BUILD_DIR}" --config "${BUILD_CONFIG}" - -# -# Run tests -# -( - cd "${BUILD_DIR}" - caplog 03_combined_testlog \ - ctest -C "${BUILD_CONFIG}" -j4 ${CAPLOG_CTEST_ARGS:-} -) +kokoro/common/cmake.sh diff --git a/kokoro/release/python/macos/config.sh b/kokoro/release/python/macos/config.sh index 91e62d6c7..250c76bc0 100644 --- a/kokoro/release/python/macos/config.sh +++ b/kokoro/release/python/macos/config.sh @@ -27,11 +27,10 @@ function pre_build { # Build protoc and protobuf libraries use_bazel.sh 5.1.1 - bazel build //:protoc - export PROTOC=$PWD/bazel-bin/protoc - mkdir src/.libs - ln -s $PWD/bazel-bin/libprotobuf.a src/.libs/libprotobuf.a - ln -s $PWD/bazel-bin/libprotobuf_lite.a src/.libs/libprotobuf-lite.a + bazel build -c opt //:protoc //pkg:protobuf //pkg:protobuf_lite + local _bazel_bin=$(bazel info -c opt bazel-bin) + export PROTOC=${_bazel_bin}/protoc + export LIBPROTOBUF=${_bazel_bin}/pkg/libprotobuf.a # Generate python dependencies. pushd python @@ -53,7 +52,8 @@ function bdist_wheel_cmd { # Modify build version pwd ls - python setup.py bdist_wheel --cpp_implementation --compile_static_extension + python setup.py build_ext --cpp_implementation -O${LIBPROTOBUF} + python setup.py bdist_wheel --cpp_implementation cp dist/*.whl $abs_wheelhouse } diff --git a/php/ext/google/protobuf/message.c b/php/ext/google/protobuf/message.c index 27199640c..83c51a494 100644 --- a/php/ext/google/protobuf/message.c +++ b/php/ext/google/protobuf/message.c @@ -601,12 +601,11 @@ PHP_METHOD(Message, __construct) { // // However, if the user created their own class derived from Message, this // will trigger an infinite construction loop and blow the stack. We - // temporarily clear create_object to break this loop (see check in + // store this `ce` in a global variable to break the cycle (see the check in // NameMap_GetMessage()). - PBPHP_ASSERT(ce->create_object == Message_create); - ce->create_object = NULL; + NameMap_EnterConstructor(ce); desc = Descriptor_GetFromClassEntry(ce); - ce->create_object = Message_create; + NameMap_ExitConstructor(ce); if (!desc) { zend_throw_exception_ex( diff --git a/php/ext/google/protobuf/package.xml b/php/ext/google/protobuf/package.xml index 0acb23c19..d1d5d87d2 100644 --- a/php/ext/google/protobuf/package.xml +++ b/php/ext/google/protobuf/package.xml @@ -10,11 +10,11 @@ protobuf-opensource@google.com yes - 2022-04-20 - + 2022-05-10 + - 3.20.1 - 3.20.1 + 3.21.0RC1 + 3.21.0 beta @@ -1283,5 +1283,20 @@ G A release. + + + 3.21.0RC1 + 3.21.0 + + + beta + beta + + 2022-05-10 + + BSD-3-Clause + + + diff --git a/php/ext/google/protobuf/protobuf.c b/php/ext/google/protobuf/protobuf.c index 68c0ef00b..c786b6eca 100644 --- a/php/ext/google/protobuf/protobuf.c +++ b/php/ext/google/protobuf/protobuf.c @@ -56,6 +56,9 @@ ZEND_BEGIN_MODULE_GLOBALS(protobuf) // Set by the user to make the descriptor pool persist between requests. zend_bool keep_descriptor_pool_after_request; + // Set by the user to make the descriptor pool persist between requests. + zend_class_entry* constructing_class; + // A upb_DefPool that we are saving for the next request so that we don't have // to rebuild it from scratch. When keep_descriptor_pool_after_request==true, // we steal the upb_DefPool from the global DescriptorPool object just before @@ -173,6 +176,7 @@ static PHP_RINIT_FUNCTION(protobuf) { zend_hash_init(&PROTOBUF_G(object_cache), 64, NULL, NULL, 0); zend_hash_init(&PROTOBUF_G(descriptors), 64, NULL, ZVAL_PTR_DTOR, 0); + PROTOBUF_G(constructing_class) = NULL; return SUCCESS; } @@ -253,7 +257,7 @@ const upb_MessageDef *NameMap_GetMessage(zend_class_entry *ce) { const upb_MessageDef *ret = zend_hash_find_ptr(&PROTOBUF_G(name_msg_cache), ce->name); - if (!ret && ce->create_object) { + if (!ret && ce->create_object && ce != PROTOBUF_G(constructing_class)) { #if PHP_VERSION_ID < 80000 zval tmp; zval zv; @@ -279,6 +283,16 @@ const upb_EnumDef *NameMap_GetEnum(zend_class_entry *ce) { return ret; } +void NameMap_EnterConstructor(zend_class_entry* ce) { + assert(!PROTOBUF_G(constructing_class)); + PROTOBUF_G(constructing_class) = ce; +} + +void NameMap_ExitConstructor(zend_class_entry* ce) { + assert(PROTOBUF_G(constructing_class) == ce); + PROTOBUF_G(constructing_class) = NULL; +} + // ----------------------------------------------------------------------------- // Module init. // ----------------------------------------------------------------------------- diff --git a/php/ext/google/protobuf/protobuf.h b/php/ext/google/protobuf/protobuf.h index 97adedb32..292a0eeca 100644 --- a/php/ext/google/protobuf/protobuf.h +++ b/php/ext/google/protobuf/protobuf.h @@ -127,7 +127,7 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_setter, 0, 0, 1) ZEND_ARG_INFO(0, value) ZEND_END_ARG_INFO() -#define PHP_PROTOBUF_VERSION "3.20.1" +#define PHP_PROTOBUF_VERSION "3.21.0RC1" // ptr -> PHP object cache. This is a weak map that caches lazily-created // wrapper objects around upb types: @@ -155,6 +155,8 @@ void NameMap_AddMessage(const upb_MessageDef *m); void NameMap_AddEnum(const upb_EnumDef *m); const upb_MessageDef *NameMap_GetMessage(zend_class_entry *ce); const upb_EnumDef *NameMap_GetEnum(zend_class_entry *ce); +void NameMap_EnterConstructor(zend_class_entry* ce); +void NameMap_ExitConstructor(zend_class_entry* ce); // Add this descriptor object to the global list of descriptors that will be // kept alive for the duration of the request but destroyed when the request diff --git a/php/src/Google/Protobuf/Internal/MapField.php b/php/src/Google/Protobuf/Internal/MapField.php index d44377a17..d413c6d90 100644 --- a/php/src/Google/Protobuf/Internal/MapField.php +++ b/php/src/Google/Protobuf/Internal/MapField.php @@ -131,7 +131,7 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable * * This will also be called for: $ele = $arr[$key] * - * @param int|bool|string $key The key of the element to be fetched. + * @param int|string $key The key of the element to be fetched. * @return object The stored element at given key. * @throws \ErrorException Invalid type for index. * @throws \ErrorException Non-existing index. @@ -148,7 +148,7 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable * * This will also be called for: $arr[$key] = $value * - * @param object $key The key of the element to be fetched. + * @param int|string $key The key of the element to be fetched. * @param object $value The element to be assigned. * @return void * @throws \ErrorException Invalid type for key. @@ -211,7 +211,7 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable * * This will also be called for: unset($arr) * - * @param object $key The key of the element to be removed. + * @param int|string $key The key of the element to be removed. * @return void * @throws \ErrorException Invalid type for key. * @todo need to add return type void (require update php version to 7.1) @@ -228,7 +228,7 @@ class MapField implements \ArrayAccess, \IteratorAggregate, \Countable * * This will also be called for: isset($arr) * - * @param object $key The key of the element to be removed. + * @param int|string $key The key of the element to be removed. * @return bool True if the element at the given key exists. * @throws \ErrorException Invalid type for key. */ diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index a197c74ec..f01b517b0 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -4,6 +4,7 @@ load( "pkg_attributes", "pkg_filegroup", "pkg_files", + "strip_prefix", ) load("//:protobuf_release.bzl", "package_naming") load(":build_systems.bzl", "gen_automake_file_lists", "gen_file_lists") @@ -15,18 +16,7 @@ package_naming( pkg_files( name = "wkt_protos_files", - srcs = [ - "//:any_proto", - "//:api_proto", - "//:duration_proto", - "//:empty_proto", - "//:field_mask_proto", - "//:source_context_proto", - "//:struct_proto", - "//:timestamp_proto", - "//:type_proto", - "//:wrappers_proto", - ], + srcs = ["//src/google/protobuf:well_known_type_protos"], prefix = "include/google/protobuf", visibility = ["//visibility:private"], ) @@ -40,13 +30,6 @@ pkg_files( visibility = ["//visibility:private"], ) -pkg_files( - name = "compiler_plugin_protos_files", - srcs = ["//:compiler_plugin_proto"], - prefix = "include/google/protobuf/compiler", - visibility = ["//visibility:private"], -) - ################################################################################ # Generates protoc release artifacts. ################################################################################ @@ -82,11 +65,11 @@ pkg_files( pkg_zip( name = "protoc_release", srcs = [ - ":compiler_plugin_protos_files", ":descriptor_protos_files", ":protoc_files", ":protoc_readme", ":wkt_protos_files", + "//src/google/protobuf/compiler:compiler_plugin_protos_files", ], package_file_name = "protoc-{version}-{platform}.zip", package_variables = ":protobuf_pkg_naming", @@ -101,10 +84,12 @@ pkg_zip( pkg_filegroup( name = "dist_common", srcs = [ + ":dist_files", "//:common_dist_files", - "//:conformance_dist_files", - "//:cpp_dist_files", # to build protoc "//benchmarks:all_dist_files", + "//build_defs:dist_files", + "//conformance:all_dist_files", + "//src:all_dist_files", "@com_google_protobuf_examples//:dist_files", ], ) @@ -113,7 +98,7 @@ pkg_filegroup( name = "cpp_srcs", srcs = [ ":dist_common", - "//:cpp_dist_files", + "//src:all_dist_files", ], ) @@ -235,7 +220,7 @@ pkg_filegroup( name = "python_srcs", srcs = [ ":dist_common", - "//:python_dist_files", + "//python:dist_files", ], ) @@ -319,7 +304,7 @@ gen_file_lists( src_libs = { # source rule: name in generated file "//:protobuf": "libprotobuf", - "//:protoc_lib": "libprotoc", + "//src/google/protobuf/compiler:protoc_lib": "libprotoc", "//:protobuf_lite": "libprotobuf_lite", }, ) @@ -331,7 +316,7 @@ gen_automake_file_lists( src_libs = { # source rule: name in generated file "//:common_dist_files": "dist_common", - "//:conformance_dist_files": "dist_conformance", + "//conformance:dist_files": "dist_conformance", "//benchmarks:all_dist_files": "dist_benchmark", "@com_google_protobuf_examples//:dist_files": "dist_example", "//:csharp_dist_files": "dist_csharp", @@ -339,7 +324,7 @@ gen_automake_file_lists( "//:objectivec_dist_files": "dist_objectivec", "//objectivec:dist_files": "dist_objectivec2", "//php:dist_files": "dist_php", - "//:python_dist_files": "dist_python", + "//python:dist_files": "dist_python", "//ruby:dist_files": "dist_ruby", }, ) @@ -351,25 +336,56 @@ gen_automake_file_lists( cc_dist_library( name = "protobuf_lite", linkopts = select({ - "//:msvc": [], + "//build_defs:config_msvc": [], "//conditions:default": ["-lpthread"], }), + tags = ["manual"], deps = [ "//:protobuf_lite", + "//src/google/protobuf:arena", + "//src/google/protobuf/io", + "//src/google/protobuf/io:io_win32", + "//src/google/protobuf/stubs:lite", ], ) cc_dist_library( name = "protobuf", linkopts = select({ - "//:msvc": [], + "//build_defs:config_msvc": [], "//conditions:default": [ "-lz", "-lpthread", ], }), + tags = ["manual"], deps = [ "//:protobuf", "//:protobuf_lite", + "//src/google/protobuf:arena", + "//src/google/protobuf/compiler:importer", + "//src/google/protobuf/io", + "//src/google/protobuf/io:gzip_stream", + "//src/google/protobuf/io:io_win32", + "//src/google/protobuf/io:printer", + "//src/google/protobuf/io:tokenizer", + "//src/google/protobuf/stubs", + "//src/google/protobuf/stubs:lite", + "//src/google/protobuf/util:delimited_message_util", + "//src/google/protobuf/util:differencer", + "//src/google/protobuf/util:field_mask_util", + "//src/google/protobuf/util:json_util", + "//src/google/protobuf/util:time_util", + "//src/google/protobuf/util:type_resolver_util", ], ) + +################################################################################ +# Distribution sources +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["*"]), + strip_prefix = strip_prefix.from_root(""), +) diff --git a/pkg/cc_dist_library.bzl b/pkg/cc_dist_library.bzl index 654fb73e3..d48e8bed8 100644 --- a/pkg/cc_dist_library.bzl +++ b/pkg/cc_dist_library.bzl @@ -51,7 +51,7 @@ def _create_archive_action( cc_toolchain.all_files, ], ), - use_default_shell_env = True, + use_default_shell_env = False, outputs = [output_file], mnemonic = "CppArchiveDist", ) diff --git a/protobuf.bzl b/protobuf.bzl index c5b8dab38..243341df3 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -58,29 +58,18 @@ def _PyOuts(srcs, use_grpc_plugin = False): ret += [s[:-len(".proto")] + "_pb2_grpc.py" for s in srcs] return ret -def _RelativeOutputPath(path, include, dest = ""): - if include == None: - return path - - if not path.startswith(include): - fail("Include path %s isn't part of the path %s." % (include, path)) - - if include and include[-1] != "/": - include = include + "/" - if dest and dest[-1] != "/": - dest = dest + "/" - - path = path[len(include):] - return dest + path +ProtoGenInfo = provider( + fields = ["srcs", "import_flags", "deps"], +) def _proto_gen_impl(ctx): """General implementation for generating protos""" srcs = ctx.files.srcs - deps = depset(direct=ctx.files.srcs) + deps = depset(direct = ctx.files.srcs) source_dir = _SourceDir(ctx) gen_dir = _GenDir(ctx).rstrip("/") import_flags = [] - + if source_dir: has_sources = any([src.is_source for src in srcs]) if has_sources: @@ -92,27 +81,34 @@ def _proto_gen_impl(ctx): if has_generated: import_flags += ["-I" + gen_dir] - import_flags = depset(direct=import_flags) + import_flags = depset(direct = import_flags) for dep in ctx.attr.deps: - if type(dep.proto.import_flags) == "list": - import_flags = depset(transitive=[import_flags], direct=dep.proto.import_flags) + dep_proto = dep[ProtoGenInfo] + if type(dep_proto.import_flags) == "list": + import_flags = depset( + transitive = [import_flags], + direct = dep_proto.import_flags, + ) else: - import_flags = depset(transitive=[import_flags, dep.proto.import_flags]) - if type(dep.proto.deps) == "list": - deps = depset(transitive=[deps], direct=dep.proto.deps) + import_flags = depset( + transitive = [import_flags, dep_proto.import_flags], + ) + if type(dep_proto.deps) == "list": + deps = depset(transitive = [deps], direct = dep_proto.deps) else: - deps = depset(transitive=[deps, dep.proto.deps]) + deps = depset(transitive = [deps, dep_proto.deps]) if not ctx.attr.gen_cc and not ctx.attr.gen_py and not ctx.executable.plugin: - return struct( - proto = struct( + return [ + ProtoGenInfo( srcs = srcs, import_flags = import_flags, deps = deps, ), - ) + ] + generated_files = [] for src in srcs: args = [] @@ -134,6 +130,8 @@ def _proto_gen_impl(ctx): outs.extend(_PyOuts([src.basename], use_grpc_plugin = use_grpc_plugin)) outs = [ctx.actions.declare_file(out, sibling = src) for out in outs] + generated_files.extend(outs) + inputs = [src] + deps.to_list() tools = [ctx.executable.protoc] if ctx.executable.plugin: @@ -186,18 +184,19 @@ def _proto_gen_impl(ctx): use_default_shell_env = True, ) - return struct( - proto = struct( + return [ + ProtoGenInfo( srcs = srcs, import_flags = import_flags, deps = deps, ), - ) + DefaultInfo(files = depset(generated_files)), + ] proto_gen = rule( attrs = { "srcs": attr.label_list(allow_files = True), - "deps": attr.label_list(providers = ["proto"]), + "deps": attr.label_list(providers = [ProtoGenInfo]), "includes": attr.string_list(), "protoc": attr.label( cfg = "exec", @@ -214,7 +213,7 @@ proto_gen = rule( "plugin_options": attr.string_list(), "gen_cc": attr.bool(), "gen_py": attr.bool(), - "outs": attr.output_list(), + "outs": attr.label_list(), }, output_to_genfiles = True, implementation = _proto_gen_impl, @@ -315,7 +314,6 @@ def cc_proto_library( plugin = grpc_cpp_plugin, plugin_language = "grpc", gen_cc = 1, - outs = outs, visibility = ["//visibility:public"], ) @@ -456,41 +454,6 @@ internal_gen_kt_protos = rule( }, ) - - -def internal_copied_filegroup(name, srcs, strip_prefix, dest, **kwargs): - """Macro to copy files to a different directory and then create a filegroup. - - This is used by the //:protobuf_python py_proto_library target to work around - an issue caused by Python source files that are part of the same Python - package being in separate directories. - - Args: - srcs: The source files to copy and add to the filegroup. - strip_prefix: Path to the root of the files to copy. - dest: The directory to copy the source files into. - **kwargs: extra arguments that will be passesd to the filegroup. - """ - outs = [_RelativeOutputPath(s, strip_prefix, dest) for s in srcs] - - native.genrule( - name = name + "_genrule", - srcs = srcs, - outs = outs, - cmd_bash = " && ".join( - ["cp $(location %s) $(location %s)" % - (s, _RelativeOutputPath(s, strip_prefix, dest)) for s in srcs]), - cmd_bat = " && ".join( - ["@copy /Y $(location %s) $(location %s) >NUL" % - (s, _RelativeOutputPath(s, strip_prefix, dest)) for s in srcs]), - ) - - native.filegroup( - name = name, - srcs = outs, - **kwargs - ) - def py_proto_library( name, srcs = [], @@ -501,6 +464,7 @@ def py_proto_library( default_runtime = "@com_google_protobuf//:protobuf_python", protoc = "@com_google_protobuf//:protoc", use_grpc_plugin = False, + testonly = None, **kargs): """Bazel rule to create a Python protobuf library from proto source files @@ -522,11 +486,11 @@ def py_proto_library( protoc: the label of the protocol compiler to generate the sources. use_grpc_plugin: a flag to indicate whether to call the Python C++ plugin when processing the proto files. + testonly: common rule attribute (see: + https://bazel.build/reference/be/common-definitions#common-attributes) **kargs: other keyword arguments that are passed to py_library. """ - outs = _PyOuts(srcs, use_grpc_plugin) - includes = [] if include != None: includes = [include] @@ -540,12 +504,12 @@ def py_proto_library( proto_gen( name = name + "_genproto", + testonly = testonly, srcs = srcs, deps = [s + "_genproto" for s in deps], includes = includes, protoc = protoc, gen_py = 1, - outs = outs, visibility = ["//visibility:public"], plugin = grpc_python_plugin, plugin_language = "grpc", @@ -555,34 +519,13 @@ def py_proto_library( py_libs = py_libs + [default_runtime] py_library( name = name, - srcs = outs + py_extra_srcs, + testonly = testonly, + srcs = [name + "_genproto"] + py_extra_srcs, deps = py_libs + deps, imports = includes, **kargs ) -def internal_protobuf_py_tests( - name, - modules = [], - **kargs): - """Bazel rules to create batch tests for protobuf internal. - - Args: - name: the name of the rule. - modules: a list of modules for tests. The macro will create a py_test for - each of the parameter with the source "google/protobuf/%s.py" - kargs: extra parameters that will be passed into the py_test. - - """ - for m in modules: - s = "python/google/protobuf/internal/%s.py" % m - py_test( - name = "py_%s" % m, - srcs = [s], - main = s, - **kargs - ) - def check_protobuf_required_bazel_version(): """For WORKSPACE files, to check the installed version of bazel. diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index eab12e33c..0f3c76aac 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -114,6 +114,6 @@ def protobuf_deps(): _github_archive( name = "upb", repo = "https://github.com/protocolbuffers/upb", - commit = "0e8772fc20e5a0a2fa1f326c79d494374871ef94", - sha256 = "2a7ccd5a8e20c5b2f6efafb305f734c7ad1cc5615706f7a35cd4fbad436d5101", + commit = "c3cfd09b0184bcbdade71a3d788df02c83e897f2", + sha256 = "4a9f79385fc0c1e3e7ba5c34220db53f956c8c42d636bafc6a563da2facf8c3f", ) diff --git a/protobuf_version.bzl b/protobuf_version.bzl index 13955260a..94e224745 100644 --- a/protobuf_version.bzl +++ b/protobuf_version.bzl @@ -1,6 +1,3 @@ -PROTOC_VERSION = '3.20.1' -PROTOBUF_JAVA_VERSION = '3.20.1' -PROTOBUF_PYTHON_VERSION = '3.20.1' - -# Remove once upb has been updated to use PROTOBUF_PYTHON_VERSION instead. -PROTOBUF_VERSION = PROTOBUF_PYTHON_VERSION +PROTOC_VERSION = '21.0-rc-1' +PROTOBUF_JAVA_VERSION = '3.21.0-rc-1' +PROTOBUF_PYTHON_VERSION = '4.21.0-rc-1' diff --git a/protoc-artifacts/pom.xml b/protoc-artifacts/pom.xml index e03216bfc..7f285cae4 100644 --- a/protoc-artifacts/pom.xml +++ b/protoc-artifacts/pom.xml @@ -8,7 +8,7 @@ com.google.protobuf protoc - 3.20.1 + 21.0-rc-1 pom Protobuf Compiler diff --git a/python/BUILD.bazel b/python/BUILD.bazel new file mode 100644 index 000000000..06c8a5dff --- /dev/null +++ b/python/BUILD.bazel @@ -0,0 +1,358 @@ +# Protobuf Python runtime +# +# See also code generation logic under /src/google/protobuf/compiler/python. +# +# Most users should depend upon public aliases in the root: +# //:protobuf_python +# //:well_known_types_py_pb2 + +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_python//python:defs.bzl", "py_library") +load("//:protobuf.bzl", "py_proto_library") +load("//build_defs:cpp_opts.bzl", "COPTS") +load(":internal.bzl", "internal_copy_files") + +py_library( + name = "protobuf_python", + data = select({ + "//conditions:default": [], + ":use_fast_cpp_protos": [ + ":google/protobuf/internal/_api_implementation.so", + ":google/protobuf/pyext/_message.so", + ], + }), + visibility = ["//:__pkg__"], + deps = [ + ":python_srcs", + ":well_known_types_py_pb2", + ], +) + +config_setting( + name = "use_fast_cpp_protos", + values = { + "define": "use_fast_cpp_protos=true", + }, +) + +py_proto_library( + name = "well_known_types_py_pb2", + srcs = [":copied_wkt_proto_files"], + include = ".", + default_runtime = "", + protoc = "//:protoc", + srcs_version = "PY2AND3", + visibility = [ + "//:__pkg__", + "@upb//:__subpackages__", + ], +) + +internal_copy_files( + name = "copied_wkt_proto_files", + srcs = [ + "//src/google/protobuf:descriptor_proto_srcs", + "//:well_known_type_protos", + ], + strip_prefix = "src", +) + +cc_binary( + name = "google/protobuf/internal/_api_implementation.so", + srcs = ["google/protobuf/internal/api_implementation.cc"], + copts = COPTS + [ + "-DPYTHON_PROTO2_CPP_IMPL_V2", + ], + linkshared = 1, + linkstatic = 1, + tags = [ + # Exclude this target from wildcard expansion (//...) because it may + # not even be buildable. It will be built if it is needed according + # to :use_fast_cpp_protos. + # https://docs.bazel.build/versions/master/be/common-definitions.html#common-attributes + "manual", + ], + deps = select({ + "//conditions:default": [], + ":use_fast_cpp_protos": ["//external:python_headers"], + }), +) + +config_setting( + name = "allow_oversize_protos", + values = { + "define": "allow_oversize_protos=true", + }, +) + +cc_binary( + name = "google/protobuf/pyext/_message.so", + srcs = glob([ + "google/protobuf/pyext/*.cc", + "google/protobuf/pyext/*.h", + ]), + copts = COPTS + [ + "-DGOOGLE_PROTOBUF_HAS_ONEOF=1", + ] + select({ + "//conditions:default": [], + ":allow_oversize_protos": ["-DPROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS=1"], + }), + includes = ["."], + linkshared = 1, + linkstatic = 1, + tags = [ + # Exclude this target from wildcard expansion (//...) because it may + # not even be buildable. It will be built if it is needed according + # to :use_fast_cpp_protos. + # https://docs.bazel.build/versions/master/be/common-definitions.html#common-attributes + "manual", + ], + deps = [ + ":proto_api", + "//:protobuf", + ] + select({ + "//conditions:default": [], + ":use_fast_cpp_protos": ["//external:python_headers"], + }), +) + +py_library( + name = "python_srcs", + srcs = glob( + [ + "google/protobuf/**/*.py", + ], + exclude = [ + "google/protobuf/internal/*_test.py", + "google/protobuf/internal/test_util.py", + "google/protobuf/internal/import_test_package/__init__.py", + ], + ), + imports = ["python"], + srcs_version = "PY2AND3", + visibility = [ + "//:__pkg__", + "@upb//:__subpackages__", + ], +) + +py_library( + name = "python_test_srcs", + srcs = glob([ + "google/protobuf/internal/*_test.py", + ]) + [ + "google/protobuf/internal/test_util.py", + "google/protobuf/internal/import_test_package/__init__.py", + ], + imports = ["python"], + srcs_version = "PY3", + visibility = [ + "//:__pkg__", + "@upb//:__subpackages__", + ], +) + +################################################################################ +# Tests +################################################################################ + +internal_copy_files( + name = "copied_test_proto_files", + testonly = 1, + srcs = [ + "//:test_proto_srcs", + "//src/google/protobuf/util:test_proto_srcs", + ], + strip_prefix = "src", +) + +py_proto_library( + name = "python_common_test_protos", + testonly = 1, + srcs = [":copied_test_proto_files"], + include = ".", + default_runtime = "", + protoc = "//:protoc", + srcs_version = "PY2AND3", + visibility = ["//:__pkg__"], + deps = [":well_known_types_py_pb2"], +) + +py_proto_library( + name = "python_specific_test_protos", + testonly = 1, + srcs = glob([ + "google/protobuf/internal/*.proto", + "google/protobuf/internal/import_test_package/*.proto", + ]), + include = ".", + default_runtime = ":protobuf_python", + protoc = "//:protoc", + srcs_version = "PY2AND3", + visibility = ["//:__pkg__"], + deps = [":python_common_test_protos"], +) + +py_library( + name = "python_test_lib", + testonly = 1, + srcs = [ + "google/protobuf/internal/import_test_package/__init__.py", + "google/protobuf/internal/test_util.py", + ], + imports = ["python"], + srcs_version = "PY2AND3", + deps = [ + ":protobuf_python", + ":python_common_test_protos", + ":python_specific_test_protos", + ], +) + +py_test( + name = "descriptor_database_test", + srcs = ["google/protobuf/internal/descriptor_database_test.py"], + deps = [":python_test_lib"], +) + +py_test( + name = "descriptor_pool_test", + srcs = ["google/protobuf/internal/descriptor_pool_test.py"], + deps = [":python_test_lib"], +) + +py_test( + name = "descriptor_test", + srcs = ["google/protobuf/internal/descriptor_test.py"], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "generator_test", + srcs = ["google/protobuf/internal/generator_test.py"], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "json_format_test", + srcs = ["google/protobuf/internal/json_format_test.py"], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "message_factory_test", + srcs = ["google/protobuf/internal/message_factory_test.py"], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "message_test", + srcs = ["google/protobuf/internal/message_test.py"], + data = glob(["testdata/golden_pickle_*"]) + [ + "//src/google/protobuf:testdata", + ], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "proto_builder_test", + srcs = ["google/protobuf/internal/proto_builder_test.py"], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "reflection_test", + srcs = ["google/protobuf/internal/reflection_test.py"], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "service_reflection_test", + srcs = ["google/protobuf/internal/service_reflection_test.py"], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "symbol_database_test", + srcs = ["google/protobuf/internal/symbol_database_test.py"], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "text_encoding_test", + srcs = ["google/protobuf/internal/text_encoding_test.py"], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "text_format_test", + srcs = ["google/protobuf/internal/text_format_test.py"], + data = ["//src/google/protobuf:testdata"], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "unknown_fields_test", + srcs = ["google/protobuf/internal/unknown_fields_test.py"], + imports = ["."], + deps = [":python_test_lib"], +) + +py_test( + name = "wire_format_test", + srcs = ["google/protobuf/internal/wire_format_test.py"], + imports = ["."], + deps = [":python_test_lib"], +) + +cc_library( + name = "proto_api", + hdrs = ["python/google/protobuf/proto_api.h"], + visibility = ["//visibility:public"], + deps = [ + "//external:python_headers", + ], +) + +################################################################################ +# Distribution files +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob([ + "google/**/*.proto", + "google/**/*.py", + "google/protobuf/internal/*.cc", + "google/protobuf/pyext/*.cc", + "google/protobuf/pyext/*.h", + ]) + [ + "BUILD.bazel", + "MANIFEST.in", + "README.md", + "google/protobuf/proto_api.h", + "google/protobuf/pyext/README", + "google/protobuf/python_protobuf.h", + "internal.bzl", + "mox.py", + "release.sh", + "setup.cfg", + "setup.py", + "stubout.py", + "tox.ini", + ], + strip_prefix = strip_prefix.from_root(""), + visibility = ["//pkg:__pkg__"], +) diff --git a/python/docs/requirements.txt b/python/docs/requirements.txt index 2b3e98925..0049dfb89 100644 --- a/python/docs/requirements.txt +++ b/python/docs/requirements.txt @@ -1,3 +1,5 @@ +googleapis-common-protos==1.56.1 +jinja2==3.0.0 sphinx==2.3.1 sphinx_rtd_theme==0.4.3 sphinxcontrib-napoleon==0.7 diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index 03f3b29ee..49039d5e1 100644 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,4 +30,4 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '3.20.1' +__version__ = '4.21.0rc1' diff --git a/python/google/protobuf/descriptor_pool.py b/python/google/protobuf/descriptor_pool.py index 911372a8b..7f84a63fe 100644 --- a/python/google/protobuf/descriptor_pool.py +++ b/python/google/protobuf/descriptor_pool.py @@ -144,9 +144,6 @@ class DescriptorPool(object): self._service_descriptors = {} self._file_descriptors = {} self._toplevel_extensions = {} - # TODO(jieluo): Remove _file_desc_by_toplevel_extension after - # maybe year 2020 for compatibility issue (with 3.4.1 only). - self._file_desc_by_toplevel_extension = {} self._top_enum_values = {} # We store extensions in two two-level mappings: The first key is the # descriptor of the message being extended, the second key is the extension @@ -331,6 +328,8 @@ class DescriptorPool(object): raise TypeError('Expected an extension descriptor.') if extension.extension_scope is None: + self._CheckConflictRegister( + extension, extension.full_name, extension.file.name) self._toplevel_extensions[extension.full_name] = extension try: @@ -372,12 +371,6 @@ class DescriptorPool(object): """ self._AddFileDescriptor(file_desc) - # TODO(jieluo): This is a temporary solution for FieldDescriptor.file. - # FieldDescriptor.file is added in code gen. Remove this solution after - # maybe 2020 for compatibility reason (with 3.4.1 only). - for extension in file_desc.extensions_by_name.values(): - self._file_desc_by_toplevel_extension[ - extension.full_name] = file_desc def _AddFileDescriptor(self, file_desc): """Adds a FileDescriptor to the pool, non-recursively. @@ -483,7 +476,7 @@ class DescriptorPool(object): pass try: - return self._file_desc_by_toplevel_extension[symbol] + return self._toplevel_extensions[symbol].file except KeyError: pass @@ -792,8 +785,6 @@ class DescriptorPool(object): file_descriptor.package, scope) file_descriptor.extensions_by_name[extension_desc.name] = ( extension_desc) - self._file_desc_by_toplevel_extension[extension_desc.full_name] = ( - file_descriptor) for desc_proto in file_proto.message_type: self._SetAllFieldTypes(file_proto.package, desc_proto, scope) diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index 9e451b45b..efd909a1b 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -33,7 +33,6 @@ __author__ = 'matthewtoia@google.com (Matt Toia)' import copy -import os import unittest import warnings @@ -415,7 +414,6 @@ class DescriptorPoolTestBase(object): field = file_json.message_types_by_name['class'].fields_by_name['int_field'] self.assertEqual(field.json_name, 'json_int') - def testEnumDefaultValue(self): """Test the default value of enums which don't start at zero.""" def _CheckDefaultValue(file_descriptor): diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index 18b784e5b..49361ee5e 100644 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -1598,6 +1598,47 @@ class Proto2Tests(TextFormatBase): self.assertEqual(23, message.message_set.Extensions[ext1].i) self.assertEqual('foo', message.message_set.Extensions[ext2].str) + # Handle Any messages inside unknown extensions. + message = any_test_pb2.TestAny() + text = ('any_value {\n' + ' [type.googleapis.com/google.protobuf.internal.TestAny] {\n' + ' [unknown_extension] {\n' + ' str: "string"\n' + ' any_value {\n' + ' [type.googleapis.com/protobuf_unittest.OneString] {\n' + ' data: "string"\n' + ' }\n' + ' }\n' + ' }\n' + ' }\n' + '}\n' + 'int32_value: 123') + text_format.Parse(text, message, allow_unknown_extension=True) + self.assertEqual(123, message.int32_value) + + # Fail if invalid Any message type url inside unknown extensions. + message = any_test_pb2.TestAny() + text = ('any_value {\n' + ' [type.googleapis.com.invalid/google.protobuf.internal.TestAny] {\n' + ' [unknown_extension] {\n' + ' str: "string"\n' + ' any_value {\n' + ' [type.googleapis.com/protobuf_unittest.OneString] {\n' + ' data: "string"\n' + ' }\n' + ' }\n' + ' }\n' + ' }\n' + '}\n' + 'int32_value: 123') + self.assertRaisesRegex( + text_format.ParseError, + '[type.googleapis.com.invalid/google.protobuf.internal.TestAny]', + text_format.Parse, + text, + message, + allow_unknown_extension=True) + def testParseBadIdentifier(self): message = unittest_pb2.TestAllTypes() text = ('optional_nested_message { "bb": 1 }') diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 162531226..b434934ee 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -40,7 +40,6 @@ #include #include -#include #include #include #include @@ -48,6 +47,7 @@ #include #include #include +#include #include #define PyString_AsStringAndSize(ob, charpp, sizep) \ diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 2c4a9573e..abfe0454e 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -51,8 +51,6 @@ #endif #include #include -#include -#include #include #include #include @@ -71,7 +69,9 @@ #include #include #include +#include #include +#include #include // clang-format off diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py index a6d8bcf64..fae5f23fe 100644 --- a/python/google/protobuf/text_format.py +++ b/python/google/protobuf/text_format.py @@ -856,9 +856,12 @@ class _Parser(object): ParseError: On text parsing problems. """ # Tokenize expects native str lines. - str_lines = ( - line if isinstance(line, str) else line.decode('utf-8') - for line in lines) + try: + str_lines = ( + line if isinstance(line, str) else line.decode('utf-8') + for line in lines) + except UnicodeDecodeError as e: + raise self._StringParseError(e) tokenizer = Tokenizer(str_lines) while not tokenizer.AtEnd(): self._MergeField(tokenizer, message) @@ -1190,10 +1193,17 @@ def _SkipField(tokenizer): tokenizer: A tokenizer to parse the field name and values. """ if tokenizer.TryConsume('['): - # Consume extension name. + # Consume extension or google.protobuf.Any type URL tokenizer.ConsumeIdentifier() + num_identifiers = 1 while tokenizer.TryConsume('.'): tokenizer.ConsumeIdentifier() + num_identifiers += 1 + # This is possibly a type URL for an Any message. + if num_identifiers == 3 and tokenizer.TryConsume('/'): + tokenizer.ConsumeIdentifier() + while tokenizer.TryConsume('.'): + tokenizer.ConsumeIdentifier() tokenizer.Consume(']') else: tokenizer.ConsumeIdentifierOrNumber() diff --git a/python/internal.bzl b/python/internal.bzl new file mode 100644 index 000000000..e9bcb27f5 --- /dev/null +++ b/python/internal.bzl @@ -0,0 +1,108 @@ +# Internal helpers for building the Python protobuf runtime. + +def _internal_copy_files_impl(ctx): + strip_prefix = ctx.attr.strip_prefix + if strip_prefix[-1] != "/": + strip_prefix += "/" + + src_dests = [] + for src in ctx.files.srcs: + if src.short_path[:len(strip_prefix)] != strip_prefix: + fail("Source does not start with %s: %s" % + (strip_prefix, src.short_path)) + dest = ctx.actions.declare_file(src.short_path[len(strip_prefix):]) + src_dests.append([src, dest]) + + if ctx.attr.is_windows: + bat_file = ctx.actions.declare_file(ctx.label.name + "_copy.bat") + ctx.actions.write( + output = bat_file, + content = "\r\n".join([ + '@copy /Y "{}" "{}" >NUL'.format( + src.path.replace("/", "\\"), + dest.path.replace("/", "\\"), + ) + for src, dest in src_dests + ]) + "\r\n", + ) + ctx.actions.run( + inputs = ctx.files.srcs, + tools = [bat_file], + outputs = [dest for src, dest in src_dests], + executable = "cmd.exe", + arguments = ["/C", bat_file.path.replace("/", "\\")], + mnemonic = "InternalCopyFile", + progress_message = "Copying files", + use_default_shell_env = True, + ) + + else: + sh_file = ctx.actions.declare_file(ctx.label.name + "_copy.sh") + ctx.actions.write( + output = sh_file, + content = "\n".join([ + 'cp -f "{}" "{}"'.format(src.path, dest.path) + for src, dest in src_dests + ]), + ) + ctx.actions.run( + inputs = ctx.files.srcs, + tools = [sh_file], + outputs = [dest for src, dest in src_dests], + executable = "bash", + arguments = [sh_file.path], + mnemonic = "InternalCopyFile", + progress_message = "Copying files", + use_default_shell_env = True, + ) + + return [ + DefaultInfo(files = depset([dest for src, dest in src_dests])), + ] + +internal_copy_files_impl = rule( + doc = """ +Implementation for internal_copy_files macro. + +This rule implements file copying, including a compatibility mode for Windows. +""", + implementation = _internal_copy_files_impl, + attrs = { + "srcs": attr.label_list(allow_files = True, providers = [DefaultInfo]), + "strip_prefix": attr.string(), + "is_windows": attr.bool(), + }, +) + +def internal_copy_files(name, srcs, strip_prefix, **kwargs): + """Copies common proto files to the python tree. + + In order for Python imports to work, generated proto interfaces under + the google.protobuf package need to be in the same directory as other + source files. This rule copies the .proto files themselves, e.g. with + strip_prefix = 'src', 'src/google/protobuf/blah.proto' could be copied + to '/google/protobuf/blah.proto'. + + (An alternative might be to implement a separate rule to generate + Python code in a different location for the sources. However, this + would be strange behavior that doesn't match any other language's proto + library generation.) + + Args: + name: the name for the rule. + srcs: the sources. + strip_prefix: the prefix to remove from each of the paths in 'srcs'. The + remainder will be used to construct the output path. + **kwargs: common rule arguments. + + """ + internal_copy_files_impl( + name = name, + srcs = srcs, + strip_prefix = strip_prefix, + is_windows = select({ + "@bazel_tools//src/conditions:host_windows": True, + "//conditions:default": False, + }), + **kwargs + ) diff --git a/python/setup.py b/python/setup.py index cbb9a5906..380c7d809 100755 --- a/python/setup.py +++ b/python/setup.py @@ -220,6 +220,59 @@ def GetOptionFromArgv(option_str): return False +def _GetFlagValues(flag_long, flag_short): + """Searches sys.argv for distutils-style flags and yields values.""" + + expect_value = flag_long.endswith('=') + flag_res = [re.compile(r'--?%s(=(.*))?' % + (flag_long[:-1] if expect_value else flag_long))] + if flag_short: + flag_res.append(re.compile(r'-%s(.*)?' % (flag_short))) + + flag_match = None + for arg in sys.argv: + # If the last arg was like '-O', check if this is the library we want. + if flag_match is not None: + yield arg + flag_match = None + continue + + for flag_re in flag_res: + m = flag_re.match(arg) + if m is None: + continue + if not expect_value: + yield arg + continue + groups = m.groups() + # Check for matches: + # --long-name=foo => ('=foo', 'foo') + # -Xfoo => ('foo') + # N.B.: if the flag is like '--long-name=', then there is a value + # (the empty string). + if groups[0] or groups[-1]: + yield groups[-1] + continue + flag_match = m + + return False + + +def HasStaticLibprotobufOpt(): + """Returns true if there is a --link-objects arg for libprotobuf.""" + + lib_re = re.compile(r'(.*[/\\])?(lib)?protobuf([.]pic)?[.](a|lib)') + for value in _GetFlagValues('link-objects=', 'O'): + if lib_re.match(value): + return True + return False + + +def HasLibraryDirsOpt(): + """Returns true if there is a --library-dirs arg.""" + return any(_GetFlagValues('library-dirs=', 'L')) + + if __name__ == '__main__': ext_module_list = [] warnings_as_errors = '--warnings_as_errors' @@ -227,13 +280,24 @@ if __name__ == '__main__': # Link libprotobuf.a and libprotobuf-lite.a statically with the # extension. Note that those libraries have to be compiled with # -fPIC for this to work. - compile_static_ext = GetOptionFromArgv('--compile_static_extension') - libraries = ['protobuf'] + compile_static_ext = HasStaticLibprotobufOpt() + if GetOptionFromArgv('--compile_static_extension'): + # FUTURE: add a warning and deprecate --compile_static_extension. + compile_static_ext = True extra_objects = None if compile_static_ext: libraries = None - extra_objects = ['../src/.libs/libprotobuf.a', - '../src/.libs/libprotobuf-lite.a'] + library_dirs = None + if not HasStaticLibprotobufOpt(): + extra_objects = ['../src/.libs/libprotobuf.a', + '../src/.libs/libprotobuf-lite.a'] + else: + libraries = ['protobuf'] + if HasLibraryDirsOpt(): + library_dirs = None + else: + library_dirs = ['../src/.libs'] + TestConformanceCmd.target = 'test_python_cpp' extra_compile_args = [] @@ -305,7 +369,7 @@ if __name__ == '__main__': libraries=libraries, extra_objects=extra_objects, extra_link_args=message_extra_link_args, - library_dirs=['../src/.libs'], + library_dirs=library_dirs, extra_compile_args=extra_compile_args, ), Extension( diff --git a/ruby/Rakefile b/ruby/Rakefile index 8b5b68d85..d8ac763e0 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -103,7 +103,7 @@ else ext.lib_dir = "lib/google" ext.cross_compile = true ext.cross_platform = [ - 'x86-mingw32', 'x64-mingw32', + 'x86-mingw32', 'x64-mingw32', 'x64-mingw-ucrt', 'x86_64-linux', 'x86-linux', 'x86_64-darwin', 'arm64-darwin', ] @@ -126,7 +126,7 @@ else task 'gem:windows' do sh "rm Gemfile.lock" require 'rake_compiler_dock' - ['x86-mingw32', 'x64-mingw32', 'x86_64-linux', 'x86-linux'].each do |plat| + ['x86-mingw32', 'x64-mingw32', 'x64-mingw-ucrt', 'x86_64-linux', 'x86-linux'].each do |plat| RakeCompilerDock.sh <<-"EOT", platform: plat bundle && \ IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0:2.6.0:2.5.0 diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index cd7aaaa97..ff55b6427 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = "google-protobuf" - s.version = "3.20.1" + s.version = "3.21.0.rc.1" git_tag = "v#{s.version.to_s.sub('.rc.', '-rc')}" # Converts X.Y.Z.rc.N to vX.Y.Z-rcN, used for the git tag s.licenses = ["BSD-3-Clause"] s.summary = "Protocol Buffers" diff --git a/ruby/pom.xml b/ruby/pom.xml index 0e7c3fd88..0d53cd1f3 100644 --- a/ruby/pom.xml +++ b/ruby/pom.xml @@ -9,7 +9,7 @@ com.google.protobuf.jruby protobuf-jruby - 3.20.1 + 3.21.0-rc-1 Protocol Buffer JRuby native extension Protocol Buffers are a way of encoding structured data in an efficient yet @@ -76,7 +76,7 @@ com.google.protobuf protobuf-java-util - 3.20.1 + 3.21.0-rc-1 org.jruby diff --git a/src/BUILD.bazel b/src/BUILD.bazel new file mode 100644 index 000000000..bd43afb8f --- /dev/null +++ b/src/BUILD.bazel @@ -0,0 +1,35 @@ +################################################################################ +# Protocol Buffers: C++ Runtime +################################################################################ + +# Most rules are under google/protobuf. This package exists for convenience. +load("@rules_pkg//:mappings.bzl", "pkg_filegroup", "pkg_files", "strip_prefix") + +pkg_files( + name = "dist_files", + srcs = glob(["**"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) + +pkg_filegroup( + name = "all_dist_files", + srcs = [ + ":dist_files", + "//src/google/protobuf:dist_files", + "//src/google/protobuf/compiler:dist_files", + "//src/google/protobuf/compiler/cpp:dist_files", + "//src/google/protobuf/compiler/csharp:dist_files", + "//src/google/protobuf/compiler/java:dist_files", + "//src/google/protobuf/compiler/objectivec:dist_files", + "//src/google/protobuf/compiler/php:dist_files", + "//src/google/protobuf/compiler/python:dist_files", + "//src/google/protobuf/compiler/ruby:dist_files", + "//src/google/protobuf/io:dist_files", + "//src/google/protobuf/stubs:dist_files", + "//src/google/protobuf/testing:dist_files", + "//src/google/protobuf/util:dist_files", + "//src/google/protobuf/util/internal:dist_files", + ], + visibility = ["//pkg:__pkg__"], +) diff --git a/src/Makefile.am b/src/Makefile.am index fc80935c0..9033a4679 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,7 +18,7 @@ else PTHREAD_DEF = endif -PROTOBUF_VERSION = 31:1:0 +PROTOBUF_VERSION = 32:0:0 if GCC # Turn on all warnings except for sign comparison (we ignore sign comparison diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel new file mode 100644 index 000000000..ac5da9049 --- /dev/null +++ b/src/google/protobuf/BUILD.bazel @@ -0,0 +1,577 @@ +################################################################################ +# Protocol Buffers: C++ Runtime and common proto files +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_proto_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") + +package( + default_visibility = [ + "//:__pkg__", # "public" targets are alias rules in //. + ], +) + +proto_library( + name = "any_proto", + srcs = ["any.proto"], + strip_import_prefix = "/src", +) + +proto_library( + name = "api_proto", + srcs = ["api.proto"], + strip_import_prefix = "/src", + deps = [ + "//:source_context_proto", + "//:type_proto", + ], +) + +proto_library( + name = "duration_proto", + srcs = ["duration.proto"], + strip_import_prefix = "/src", +) + +proto_library( + name = "empty_proto", + srcs = ["empty.proto"], + strip_import_prefix = "/src", +) + +proto_library( + name = "field_mask_proto", + srcs = ["field_mask.proto"], + strip_import_prefix = "/src", +) + +proto_library( + name = "source_context_proto", + srcs = ["source_context.proto"], + strip_import_prefix = "/src", +) + +proto_library( + name = "struct_proto", + srcs = ["struct.proto"], + strip_import_prefix = "/src", +) + +proto_library( + name = "timestamp_proto", + srcs = ["timestamp.proto"], + strip_import_prefix = "/src", +) + +proto_library( + name = "type_proto", + srcs = ["type.proto"], + strip_import_prefix = "/src", + deps = [ + "//:any_proto", + "//:source_context_proto", + ], +) + +proto_library( + name = "wrappers_proto", + srcs = ["wrappers.proto"], + strip_import_prefix = "/src", +) + +# Built-in runtime types + +proto_library( + name = "descriptor_proto", + srcs = ["descriptor.proto"], + strip_import_prefix = "/src", +) + +################################################################################ +# C++ Runtime Library +################################################################################ + +cc_library( + name = "port_def", + hdrs = [ + "port.h", + "port_def.inc", + "port_undef.inc", + ], + include_prefix = "google/protobuf", + visibility = [ + "//:__subpackages__", + "//src/google/protobuf:__subpackages__", + ], +) + +cc_library( + name = "arena", + srcs = [ + "arena.cc", + ], + hdrs = [ + "arena.h", + "arena_impl.h", + "arenaz_sampler.h", + ], + include_prefix = "google/protobuf", + visibility = [ + "//:__subpackages__", + "//src/google/protobuf:__subpackages__", + ], + deps = [ + "//src/google/protobuf/stubs:lite", + ], +) + +cc_library( + name = "protobuf_lite", + srcs = [ + "any_lite.cc", + "arenastring.cc", + "arenaz_sampler.cc", + "extension_set.cc", + "generated_enum_util.cc", + "generated_message_tctable_lite.cc", + "generated_message_util.cc", + "implicit_weak_message.cc", + "inlined_string_field.cc", + "map.cc", + "message_lite.cc", + "parse_context.cc", + "repeated_field.cc", + "repeated_ptr_field.cc", + "wire_format_lite.cc", + ], + hdrs = glob([ + "**/*.h", + "**/*.inc", + ]), + copts = COPTS + select({ + "//build_defs:config_msvc": [], + "//conditions:default": [ + "-Wno-error", + ], + }), + include_prefix = "google/protobuf", + linkopts = LINK_OPTS, + visibility = [ + "//:__pkg__", + "//src/google/protobuf:__subpackages__", + ], + # In Bazel 6.0+, these will be `interface_deps`: + deps = [ + ":arena", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs:lite", + ], +) + +cc_library( + name = "protobuf", + srcs = [ + "any.cc", + "any.pb.cc", + "api.pb.cc", + "descriptor.cc", + "descriptor.pb.cc", + "descriptor_database.cc", + "duration.pb.cc", + "dynamic_message.cc", + "empty.pb.cc", + "extension_set_heavy.cc", + "field_mask.pb.cc", + "generated_message_bases.cc", + "generated_message_reflection.cc", + "generated_message_tctable_full.cc", + "map_field.cc", + "message.cc", + "reflection_ops.cc", + "service.cc", + "source_context.pb.cc", + "struct.pb.cc", + "text_format.cc", + "timestamp.pb.cc", + "type.pb.cc", + "unknown_field_set.cc", + "wire_format.cc", + "wrappers.pb.cc", + ], + hdrs = glob([ + "**/*.h", + "**/*.inc", + ]), + copts = COPTS, + include_prefix = "google/protobuf", + linkopts = LINK_OPTS, + visibility = [ + "//:__pkg__", + "//src/google/protobuf:__subpackages__", + ], + deps = [ + ":protobuf_lite", + "//src/google/protobuf/io", + "//src/google/protobuf/io:gzip_stream", + "//src/google/protobuf/io:printer", + "//src/google/protobuf/io:tokenizer", + "//src/google/protobuf/stubs", + ], +) + +# This provides just the header files for use in projects that need to build +# shared libraries for dynamic loading. This target is available until Bazel +# adds native support for such use cases. +# TODO(keveman): Remove this target once the support gets added to Bazel. +cc_library( + name = "protobuf_headers", + hdrs = glob([ + "**/*.h", + "**/*.inc", + ]), +) + +filegroup( + name = "well_known_type_protos", + srcs = [ + "any.proto", + "api.proto", + "duration.proto", + "empty.proto", + "field_mask.proto", + "source_context.proto", + "struct.proto", + "timestamp.proto", + "type.proto", + "wrappers.proto", + ], + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "descriptor_proto_srcs", + srcs = ["descriptor.proto"], + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "testdata", + srcs = glob(["testdata/**/*"]) + [ + "descriptor.cc", + ], + visibility = [ + "//:__subpackages__", + "@upb//:__subpackages__", + ], +) + +proto_library( + name = "lite_test_protos", + srcs = [ + "map_lite_unittest.proto", + "unittest_import_lite.proto", + "unittest_import_public_lite.proto", + "unittest_lite.proto", + ], + strip_import_prefix = "/src", + visibility = ["//:__subpackages__"], + deps = [ + ":any_proto", + ":api_proto", + ":descriptor_proto", + ":duration_proto", + ":empty_proto", + ":field_mask_proto", + ":source_context_proto", + ":struct_proto", + ":timestamp_proto", + ":type_proto", + ":wrappers_proto", + ], +) + +filegroup( + name = "test_proto_srcs", + srcs = [ + "any_test.proto", + "map_proto2_unittest.proto", + "map_unittest.proto", + "unittest.proto", + "unittest_arena.proto", + "unittest_custom_options.proto", + "unittest_drop_unknown_fields.proto", + "unittest_embed_optimize_for.proto", + "unittest_empty.proto", + "unittest_enormous_descriptor.proto", + "unittest_import.proto", + "unittest_import_public.proto", + "unittest_lazy_dependencies.proto", + "unittest_lazy_dependencies_custom_option.proto", + "unittest_lazy_dependencies_enum.proto", + "unittest_lite_imports_nonlite.proto", + "unittest_mset.proto", + "unittest_mset_wire_format.proto", + "unittest_no_field_presence.proto", + "unittest_no_generic_services.proto", + "unittest_optimize_for.proto", + "unittest_preserve_unknown_enum.proto", + "unittest_preserve_unknown_enum2.proto", + "unittest_proto3.proto", + "unittest_proto3_arena.proto", + "unittest_proto3_arena_lite.proto", + "unittest_proto3_lite.proto", + "unittest_proto3_optional.proto", + "unittest_well_known_types.proto", + ], + visibility = ["//:__subpackages__"], +) + +proto_library( + name = "test_protos", + srcs = [":test_proto_srcs"], + strip_import_prefix = "/src", + visibility = ["//:__subpackages__"], + deps = [ + ":any_proto", + ":api_proto", + ":descriptor_proto", + ":duration_proto", + ":empty_proto", + ":field_mask_proto", + ":source_context_proto", + ":struct_proto", + ":timestamp_proto", + ":type_proto", + ":wrappers_proto", + ], +) + +proto_library( + name = "generic_test_protos", + srcs = [ + "map_proto2_unittest.proto", + "map_unittest.proto", + "unittest.proto", + "unittest_arena.proto", + "unittest_custom_options.proto", + "unittest_drop_unknown_fields.proto", + "unittest_embed_optimize_for.proto", + "unittest_empty.proto", + "unittest_enormous_descriptor.proto", + "unittest_import.proto", + "unittest_import_public.proto", + "unittest_lazy_dependencies.proto", + "unittest_lazy_dependencies_custom_option.proto", + "unittest_lazy_dependencies_enum.proto", + "unittest_lite_imports_nonlite.proto", + "unittest_mset.proto", + "unittest_mset_wire_format.proto", + "unittest_no_field_presence.proto", + "unittest_no_generic_services.proto", + "unittest_optimize_for.proto", + "unittest_preserve_unknown_enum.proto", + "unittest_preserve_unknown_enum2.proto", + "unittest_proto3.proto", + "unittest_proto3_arena.proto", + "unittest_proto3_arena_lite.proto", + "unittest_proto3_lite.proto", + "unittest_proto3_optional.proto", + "unittest_well_known_types.proto", + ], + strip_import_prefix = "/src", + visibility = ["//:__subpackages__"], + deps = [ + ":any_proto", + ":api_proto", + ":descriptor_proto", + ":duration_proto", + ":empty_proto", + ":field_mask_proto", + ":source_context_proto", + ":struct_proto", + ":timestamp_proto", + ":type_proto", + ":wrappers_proto", + ], +) + +proto_library( + name = "test_messages_proto2_proto", + srcs = ["test_messages_proto2.proto"], + strip_import_prefix = "/src", +) + +proto_library( + name = "test_messages_proto3_proto", + srcs = ["test_messages_proto3.proto"], + strip_import_prefix = "/src", + deps = [ + ":any_proto", + ":duration_proto", + ":field_mask_proto", + ":struct_proto", + ":timestamp_proto", + ":wrappers_proto", + ], +) + +cc_proto_library( + name = "cc_lite_test_protos", + deps = [":lite_test_protos"], +) + +cc_proto_library( + name = "cc_test_protos", + visibility = ["//src/google/protobuf:__subpackages__"], + deps = [":test_protos"], +) + +# Filegroup for golden comparison test: +filegroup( + name = "descriptor_cc_srcs", + testonly = 1, + data = [ + "descriptor.pb.cc", + "descriptor.pb.h", + ], + visibility = ["//src/google/protobuf/compiler/cpp:__pkg__"], +) + +cc_library( + name = "test_util", + testonly = 1, + srcs = [ + "arena_test_util.cc", + "map_lite_test_util.cc", + "reflection_tester.cc", + "test_util.cc", + "test_util_lite.cc", + ], + hdrs = [ + "arena_test_util.h", + "map_lite_test_util.h", + "map_test_util.h", + "map_test_util_impl.h", + "test_util.h", + "test_util2.h", + "test_util_lite.h", + ], + copts = COPTS + select({ + "//build_defs:config_msvc": [], + "//conditions:default": [ + "-Wno-error=sign-compare", + ], + }), + strip_include_prefix = "/src", + textual_hdrs = [ + "map_test_util.inc", + "test_util.inc", + ], + visibility = ["//:__subpackages__"], + deps = [ + ":cc_lite_test_protos", + ":cc_test_protos", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "protobuf_test", + srcs = [ + "any_test.cc", + "arena_unittest.cc", + "arenastring_unittest.cc", + "arenaz_sampler_test.cc", + "descriptor_database_unittest.cc", + "descriptor_unittest.cc", + "drop_unknown_fields_test.cc", + "dynamic_message_unittest.cc", + "extension_set_unittest.cc", + "generated_message_reflection_unittest.cc", + "generated_message_tctable_lite_test.cc", + "inlined_string_field_unittest.cc", + "map_field_test.cc", + "map_test.cc", + "map_test.inc", + "message_unittest.cc", + "message_unittest.inc", + "no_field_presence_test.cc", + "preserve_unknown_enum_test.cc", + "proto3_arena_lite_unittest.cc", + "proto3_arena_unittest.cc", + "proto3_lite_unittest.cc", + "proto3_lite_unittest.inc", + "reflection_ops_unittest.cc", + "repeated_field_reflection_unittest.cc", + "repeated_field_unittest.cc", + "text_format_unittest.cc", + "unknown_field_set_unittest.cc", + "well_known_types_unittest.cc", + "wire_format_unittest.cc", + "wire_format_unittest.inc", + ], + copts = COPTS + select({ + "//build_defs:config_msvc": [], + "//conditions:default": [ + "-Wno-deprecated-declarations", + "-Wno-error=sign-compare", + ], + }), + data = [ + ":testdata", + ] + glob([ + "**/*", + ]), + linkopts = LINK_OPTS, + deps = [ + ":cc_lite_test_protos", + ":cc_test_protos", + ":protobuf", + ":test_util", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) +################################################################################ +# Helper targets for Kotlin tests +################################################################################ + +proto_library( + name = "kt_unittest_protos", + srcs = [ + "map_proto2_unittest.proto", + "unittest.proto", + "unittest_import.proto", + "unittest_import_public.proto", + ], + strip_import_prefix = "/src", + visibility = ["//java/kotlin:__pkg__"], +) + +proto_library( + name = "kt_proto3_unittest_protos", + srcs = [ + "unittest_import.proto", + "unittest_import_public.proto", + "unittest_proto3.proto", + ], + strip_import_prefix = "/src", + visibility = [ + "//java/kotlin:__pkg__", + "//java/kotlin-lite:__pkg__", + ], +) + +################################################################################ +# Packaging rules +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/any.pb.h b/src/google/protobuf/any.pb.h index 8ef040c89..b3b4e6a16 100644 --- a/src/google/protobuf/any.pb.h +++ b/src/google/protobuf/any.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/api.pb.h b/src/google/protobuf/api.pb.h index 86adb0c42..2f7d90516 100644 --- a/src/google/protobuf/api.pb.h +++ b/src/google/protobuf/api.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/arenaz_sampler.cc b/src/google/protobuf/arenaz_sampler.cc index 0eac693d9..7383e3eb3 100644 --- a/src/google/protobuf/arenaz_sampler.cc +++ b/src/google/protobuf/arenaz_sampler.cc @@ -97,15 +97,12 @@ void RecordResetSlow(ThreadSafeArenaStats* info) { void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t requested, size_t allocated, size_t wasted) { + info->num_allocations.fetch_add(1, std::memory_order_relaxed); info->bytes_requested.fetch_add(requested, std::memory_order_relaxed); info->bytes_allocated.fetch_add(allocated, std::memory_order_relaxed); info->bytes_wasted.fetch_add(wasted, std::memory_order_relaxed); - info->num_allocations.fetch_add(1, std::memory_order_relaxed); - const uint64_t tid = (1ULL << (GetCachedTID() % 63)); - const uint64_t thread_ids = info->thread_ids.load(std::memory_order_relaxed); - if (!(thread_ids & tid)) { - info->thread_ids.store(thread_ids | tid, std::memory_order_relaxed); - } + const uint64_t tid = 1ULL << (GetCachedTID() % 63); + info->thread_ids.fetch_or(tid, std::memory_order_relaxed); } ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) { diff --git a/src/google/protobuf/arenaz_sampler.h b/src/google/protobuf/arenaz_sampler.h index b04b0cc67..76698a26a 100644 --- a/src/google/protobuf/arenaz_sampler.h +++ b/src/google/protobuf/arenaz_sampler.h @@ -70,8 +70,10 @@ struct ThreadSafeArenaStats std::atomic bytes_wasted; // Records the largest size an arena ever had. Maintained across resets. std::atomic max_bytes_allocated; - // Bit i when set to 1 indicates that a thread with tid % 63 = i accessed the - // underlying arena. The field is maintained across resets. + // Bit `i` is set to 1 indicates that a thread with `tid % 63 = i` accessed + // the underlying arena. We use `% 63` as a rudimentary hash to ensure some + // bit mixing for thread-ids; `% 64` would only grab the low bits and might + // create sampling artifacts. Maintained across resets. std::atomic thread_ids; // All of the fields below are set by `PrepareForSampling`, they must not diff --git a/src/google/protobuf/compiler/BUILD.bazel b/src/google/protobuf/compiler/BUILD.bazel new file mode 100644 index 000000000..8da74f10a --- /dev/null +++ b/src/google/protobuf/compiler/BUILD.bazel @@ -0,0 +1,295 @@ +################################################################################ +# Protocol Buffers Compiler +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +load( + "@rules_pkg//:mappings.bzl", + "pkg_attributes", + "pkg_filegroup", + "pkg_files", + "strip_prefix", +) +load("@rules_proto//proto:defs.bzl", "proto_library") +load("//build_defs:cpp_opts.bzl", "COPTS") + +proto_library( + name = "plugin_proto", + srcs = ["plugin.proto"], + visibility = ["//:__pkg__"], + deps = ["//:descriptor_proto"], +) + +cc_library( + name = "importer", + srcs = [ + "importer.cc", + "parser.cc", + ], + hdrs = [ + "importer.h", + "parser.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/compiler", + visibility = ["//visibility:public"], + deps = ["//src/google/protobuf"], +) + +cc_library( + name = "code_generator", + srcs = [ + "code_generator.cc", + "plugin.cc", + "plugin.pb.cc", + ], + hdrs = [ + "code_generator.h", + "plugin.h", + "plugin.pb.h", + "scc.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/compiler", + visibility = ["//visibility:public"], + deps = [ + "//:protobuf", + ], +) + +cc_library( + name = "command_line_interface", + srcs = [ + "command_line_interface.cc", + "subprocess.cc", + "zip_writer.cc", + ], + hdrs = [ + "command_line_interface.h", + "subprocess.h", + "zip_writer.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/compiler", + visibility = ["//visibility:public"], + deps = [ + ":code_generator", + ":importer", + "//:protobuf", + ], +) + +cc_library( + name = "protoc_lib", + srcs = [ + "main.cc", + ], + copts = COPTS, + visibility = [ + "//:__pkg__", + "//pkg:__pkg__", + ], + deps = [ + ":code_generator", + ":command_line_interface", + ":importer", + "//:protobuf", + "//src/google/protobuf/compiler/cpp", + "//src/google/protobuf/compiler/csharp", + "//src/google/protobuf/compiler/java", + "//src/google/protobuf/compiler/objectivec", + "//src/google/protobuf/compiler/php", + "//src/google/protobuf/compiler/python", + "//src/google/protobuf/compiler/ruby", + ], +) + +# Note: this is an alias for now. In the future, this rule will become the +# cc_binary for protoc, and //:protoc will become an alias. +alias( + name = "protoc", + actual = "//:protoc", + visibility = ["//visibility:public"], +) + +################################################################################ +# Tests and support libraries +################################################################################ + +cc_library( + name = "annotation_test_util", + testonly = 1, + srcs = ["annotation_test_util.cc"], + hdrs = ["annotation_test_util.h"], + copts = COPTS, + strip_include_prefix = "/src", + visibility = ["//visibility:public"], + deps = [ + ":code_generator", + ":command_line_interface", + "//:protobuf", + "//src/google/protobuf/io", + "//src/google/protobuf/testing", + ], +) + +################################################################################ +# Tests +################################################################################ + +filegroup( + name = "plugin_proto_srcs", + testonly = 1, + srcs = [ + "plugin.pb.cc", + "plugin.pb.h", + "plugin.proto", + ], + visibility = [ + "//src/google/protobuf/compiler/cpp:__pkg__", + ], +) + +cc_library( + name = "mock_code_generator", + testonly = 1, + srcs = ["mock_code_generator.cc"], + hdrs = ["mock_code_generator.h"], + copts = COPTS, + strip_include_prefix = "/src", + deps = [ + ":code_generator", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + "//src/google/protobuf/testing", + ], +) + +cc_binary( + name = "test_plugin", + testonly = 1, + srcs = ["test_plugin.cc"], + copts = COPTS, + deps = [ + ":code_generator", + ":mock_code_generator", + ], +) + +cc_test( + name = "command_line_interface_unittest", + srcs = ["command_line_interface_unittest.cc"], + copts = COPTS + select({ + "//build_defs:config_msvc": [], + "//conditions:default": [ + "-Wno-deprecated", + "-Wno-deprecated-declarations", + ], + }) + [ + "-DGOOGLE_PROTOBUF_TEST_PLUGIN_PATH=\\\"src/google/protobuf/compiler/test_plugin\\\"", + ], + data = [ + ":test_plugin", + "//:test_proto_srcs", + "//src/google/protobuf:testdata", + ], + deps = [ + ":code_generator", + ":command_line_interface", + ":mock_code_generator", + "//:protobuf", + "//src/google/protobuf:cc_test_protos", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "importer_unittest", + srcs = ["importer_unittest.cc"], + copts = COPTS, + deps = [ + ":importer", + "//:protobuf", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "parser_unittest", + srcs = ["parser_unittest.cc"], + copts = COPTS + select({ + "//build_defs:config_msvc": [], + "//conditions:default": [ + "-Wno-deprecated", + "-Wno-deprecated-declarations", + ], + }), + deps = [ + ":importer", + "//:protobuf", + "//src/google/protobuf:cc_test_protos", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +################################################################################ +# Generates protoc release artifacts. +################################################################################ + +genrule( + name = "protoc_readme", + outs = ["readme.txt"], + cmd = """ +echo "Protocol Buffers - Google's data interchange format +Copyright 2008 Google Inc. +https://developers.google.com/protocol-buffers/ +This package contains a precompiled binary version of the protocol buffer +compiler (protoc). This binary is intended for users who want to use Protocol +Buffers in languages other than C++ but do not want to compile protoc +themselves. To install, simply place this binary somewhere in your PATH. +If you intend to use the included well known types then don't forget to +copy the contents of the 'include' directory somewhere as well, for example +into '/usr/local/include/'. +Please refer to our official github site for more installation instructions: + https://github.com/protocolbuffers/protobuf" > $@ + """, + visibility = ["//:__pkg__"], +) + +pkg_files( + name = "compiler_plugin_protos_files", + srcs = ["plugin.proto"], + prefix = "include/google/protobuf/compiler", + visibility = ["//pkg:__pkg__"], +) + +pkg_files( + name = "protoc_files", + srcs = [":protoc"], + attributes = pkg_attributes(mode = "0555"), + prefix = "bin/", + visibility = ["//:__pkg__"], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/compiler/command_line_interface_unittest.cc b/src/google/protobuf/compiler/command_line_interface_unittest.cc index f48135ecb..99fa0ace3 100644 --- a/src/google/protobuf/compiler/command_line_interface_unittest.cc +++ b/src/google/protobuf/compiler/command_line_interface_unittest.cc @@ -1217,7 +1217,8 @@ TEST_F(CommandLineInterfaceTest, InsertWithAnnotationFixup) { "--plug_out=insert_endlines=test_generator,test_plugin:$tmpdir " "--proto_path=$tmpdir foo.proto"); - ExpectNoErrors(); + ExpectWarningSubstring( + "foo.proto:2:36: warning: Message name should be in UpperCamelCase."); CheckGeneratedAnnotations("test_generator", "foo.proto"); CheckGeneratedAnnotations("test_plugin", "foo.proto"); } @@ -2371,6 +2372,21 @@ TEST_F(CommandLineInterfaceTest, Warnings) { ExpectErrorSubstring("foo.proto:2:1: warning: Import bar.proto is unused."); } +TEST_F(CommandLineInterfaceTest, ParserWarnings) { + // Test that parser warnings are propagated. See #9343. + + CreateTempFile("foo.proto", + "syntax = \"proto2\";\n" + "message bad_to_the_bone {};\n"); + + Run("protocol_compiler --test_out=$tmpdir " + "--proto_path=$tmpdir foo.proto"); + ExpectCapturedStderrSubstringWithZeroReturnCode( + "foo.proto:2:25: warning: Message name should be in UpperCamelCase. " + "Found: bad_to_the_bone. " + "See https://developers.google.com/protocol-buffers/docs/style"); +} + // ------------------------------------------------------------------- // Flag parsing tests @@ -2681,7 +2697,7 @@ class EncodeDecodeTest : public testing::TestWithParam { TEST_P(EncodeDecodeTest, Encode) { RedirectStdinFromFile(TestUtil::GetTestDataPath( - "net/proto2/internal/" + "third_party/protobuf/" "testdata/text_format_unittest_data_oneof_implemented.txt")); std::string args; if (GetParam() != DESCRIPTOR_SET_IN) { @@ -2690,20 +2706,18 @@ TEST_P(EncodeDecodeTest, Encode) { } EXPECT_TRUE(Run(args + " --encode=protobuf_unittest.TestAllTypes")); ExpectStdoutMatchesBinaryFile(TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_message_oneof_implemented")); - ExpectStderrMatchesText(""); + "third_party/protobuf/testdata/golden_message_oneof_implemented")); } TEST_P(EncodeDecodeTest, Decode) { RedirectStdinFromFile(TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_message_oneof_implemented")); + "third_party/protobuf/testdata/golden_message_oneof_implemented")); EXPECT_TRUE( Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") + " --decode=protobuf_unittest.TestAllTypes")); ExpectStdoutMatchesTextFile(TestUtil::GetTestDataPath( - "net/proto2/internal/" + "third_party/protobuf/" "testdata/text_format_unittest_data_oneof_implemented.txt")); - ExpectStderrMatchesText(""); } TEST_P(EncodeDecodeTest, Partial) { @@ -2712,7 +2726,7 @@ TEST_P(EncodeDecodeTest, Partial) { Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") + " --encode=protobuf_unittest.TestRequired")); ExpectStdoutMatchesText(""); - ExpectStderrMatchesText( + ExpectStderrContainsText( "warning: Input message is missing required fields: a, b, c\n"); } @@ -2736,7 +2750,7 @@ TEST_P(EncodeDecodeTest, UnknownType) { Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") + " --encode=NoSuchType")); ExpectStdoutMatchesText(""); - ExpectStderrMatchesText("Type not defined: NoSuchType\n"); + ExpectStderrContainsText("Type not defined: NoSuchType\n"); } TEST_P(EncodeDecodeTest, ProtoParseError) { @@ -2750,7 +2764,7 @@ TEST_P(EncodeDecodeTest, ProtoParseError) { TEST_P(EncodeDecodeTest, EncodeDeterministicOutput) { RedirectStdinFromFile(TestUtil::GetTestDataPath( - "net/proto2/internal/" + "third_party/protobuf/" "testdata/text_format_unittest_data_oneof_implemented.txt")); std::string args; if (GetParam() != DESCRIPTOR_SET_IN) { @@ -2760,13 +2774,12 @@ TEST_P(EncodeDecodeTest, EncodeDeterministicOutput) { EXPECT_TRUE(Run( args + " --encode=protobuf_unittest.TestAllTypes --deterministic_output")); ExpectStdoutMatchesBinaryFile(TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_message_oneof_implemented")); - ExpectStderrMatchesText(""); + "third_party/protobuf/testdata/golden_message_oneof_implemented")); } TEST_P(EncodeDecodeTest, DecodeDeterministicOutput) { RedirectStdinFromFile(TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_message_oneof_implemented")); + "third_party/protobuf/testdata/golden_message_oneof_implemented")); EXPECT_FALSE( Run(TestUtil::MaybeTranslatePath("net/proto2/internal/unittest.proto") + " --decode=protobuf_unittest.TestAllTypes --deterministic_output")); diff --git a/src/google/protobuf/compiler/cpp/BUILD.bazel b/src/google/protobuf/compiler/cpp/BUILD.bazel new file mode 100644 index 000000000..37e31e803 --- /dev/null +++ b/src/google/protobuf/compiler/cpp/BUILD.bazel @@ -0,0 +1,198 @@ +################################################################################ +# Protocol Buffers Compiler - C++ code generator +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_proto_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("//build_defs:cpp_opts.bzl", "COPTS") + +cc_library( + name = "cpp", + srcs = [ + "enum.cc", + "enum_field.cc", + "extension.cc", + "field.cc", + "file.cc", + "generator.cc", + "helpers.cc", + "map_field.cc", + "message.cc", + "message_field.cc", + "padding_optimizer.cc", + "parse_function_generator.cc", + "primitive_field.cc", + "service.cc", + "string_field.cc", + ], + hdrs = [ + "enum.h", + "enum_field.h", + "extension.h", + "field.h", + "file.h", + "generator.h", + "helpers.h", + "map_field.h", + "message.h", + "message_field.h", + "message_layout_helper.h", + "names.h", + "options.h", + "padding_optimizer.h", + "parse_function_generator.h", + "primitive_field.h", + "service.h", + "string_field.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/compiler/cpp", + visibility = ["//src/google/protobuf/compiler:__pkg__"], + deps = [ + "//:protobuf", + "//src/google/protobuf/compiler:code_generator", + ], +) + +proto_library( + name = "test_bad_identifiers_proto", + testonly = 1, + srcs = ["test_bad_identifiers.proto"], + strip_import_prefix = "/src", +) + +cc_proto_library( + name = "test_bad_identifiers_cc_proto", + testonly = 1, + deps = [":test_bad_identifiers_proto"], +) + +proto_library( + name = "test_large_enum_value_proto", + testonly = 1, + srcs = ["test_large_enum_value.proto"], + strip_import_prefix = "/src", +) + +cc_proto_library( + name = "test_large_enum_value_cc_proto", + testonly = 1, + deps = [":test_large_enum_value_proto"], +) + +cc_library( + name = "unittest_lib", + hdrs = [ + "unittest.h", + "unittest.inc", + ], + strip_include_prefix = "/src", +) + +cc_test( + name = "unittest", + srcs = ["unittest.cc"], + copts = COPTS, + data = [ + "//:test_proto_srcs", + "//src/google/protobuf:testdata", + ], + deps = [ + ":cpp", + ":test_bad_identifiers_cc_proto", + ":unittest_lib", + "//:protobuf", + "//src/google/protobuf:cc_test_protos", + "//src/google/protobuf/compiler:importer", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "bootstrap_unittest", + srcs = ["bootstrap_unittest.cc"], + data = [ + "//:well_known_type_protos", + "//src/google/protobuf:descriptor_cc_srcs", + "//src/google/protobuf:descriptor_proto_srcs", + "//src/google/protobuf:testdata", + "//src/google/protobuf/compiler:plugin_proto_srcs", + ], + deps = [ + ":cpp", + "//:protobuf", + "//src/google/protobuf/compiler:importer", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "message_size_unittest", + srcs = ["message_size_unittest.cc"], + deps = [ + "//:protobuf", + "//src/google/protobuf:cc_test_protos", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "metadata_test", + srcs = ["metadata_test.cc"], + deps = [ + ":cpp", + "//:protobuf", + "//src/google/protobuf/compiler:annotation_test_util", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "move_unittest", + srcs = ["move_unittest.cc"], + copts = COPTS, + deps = [ + "//:protobuf", + "//src/google/protobuf:cc_test_protos", + "//src/google/protobuf:test_util", + "//src/google/protobuf/stubs:lite", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "plugin_unittest", + srcs = ["plugin_unittest.cc"], + deps = [ + ":cpp", + "//:protobuf", + "//src/google/protobuf/compiler:command_line_interface", + "//src/google/protobuf/io", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/compiler/cpp/message.cc b/src/google/protobuf/compiler/cpp/message.cc index 69069dacd..d3190fabc 100644 --- a/src/google/protobuf/compiler/cpp/message.cc +++ b/src/google/protobuf/compiler/cpp/message.cc @@ -2101,7 +2101,9 @@ void MessageGenerator::GenerateClassMethods(io::Printer* printer) { if (!has_bit_indices_.empty()) { format( "using HasBits = " - "decltype(std::declval<$classname$>().$has_bits$);\n"); + "decltype(std::declval<$classname$>().$has_bits$);\n" + "static constexpr int32_t kHasBitsOffset =\n" + " 8 * PROTOBUF_FIELD_OFFSET($classname$, _impl_._has_bits_);\n"); } for (auto field : FieldRange(descriptor_)) { field_generators_.get(field).GenerateInternalAccessorDeclarations(printer); diff --git a/src/google/protobuf/compiler/cpp/parse_function_generator.cc b/src/google/protobuf/compiler/cpp/parse_function_generator.cc index 0f1d767c8..91ceeeb25 100644 --- a/src/google/protobuf/compiler/cpp/parse_function_generator.cc +++ b/src/google/protobuf/compiler/cpp/parse_function_generator.cc @@ -47,6 +47,15 @@ namespace { using google::protobuf::internal::WireFormat; using google::protobuf::internal::WireFormatLite; +bool UseDirectTcParserTable(const FieldDescriptor* field, + const Options& options) { + auto* m = field->message_type(); + return !m->options().message_set_wire_format() && + m->file()->options().optimize_for() != FileOptions::CODE_SIZE && + !HasSimpleBaseClass(m, options) && !HasTracker(m, options) + ; +} + std::vector GetOrderedFields( const Descriptor* descriptor, const Options& options) { std::vector ordered_fields; @@ -324,10 +333,17 @@ TailCallTableInfo::TailCallTableInfo( // Lazy fields are handled by the generated fallback function. } else { field_entries.back().aux_idx = aux_entries.size(); - const Descriptor* field_type = field->message_type(); - aux_entries.push_back(StrCat( - "reinterpret_cast(&", QualifiedDefaultInstanceName(field_type, options), ")")); + if (UseDirectTcParserTable(field, options)) { + const Descriptor* field_type = field->message_type(); + aux_entries.push_back( + StrCat("::_pbi::TcParser::GetTable<", + QualifiedClassName(field_type, options), ">()")); + } else { + const Descriptor* field_type = field->message_type(); + aux_entries.push_back( + StrCat("::_pbi::FieldAuxDefaultMessage{}, &", + QualifiedDefaultInstanceName(field_type, options))); + } } } else if (field->type() == FieldDescriptor::TYPE_ENUM && !HasPreservingUnknownEnumSemantics(field)) { @@ -518,6 +534,9 @@ bool ParseFunctionGenerator::should_generate_tctable() const { if (options_.tctable_mode == Options::kTCTableNever) { return false; } + if (HasSimpleBaseClass(descriptor_, options_)) { + return false; + } return true; } @@ -547,7 +566,7 @@ void ParseFunctionGenerator::GenerateTailcallFallbackFunction( if (num_hasbits_ > 0) { // Sync hasbits - format("typed_msg->_impl_._has_bits_[0] = hasbits;\n"); + format("typed_msg->_impl_._has_bits_[0] |= hasbits;\n"); } format("uint32_t tag = data.tag();\n"); @@ -604,6 +623,7 @@ void ParseFunctionGenerator::GenerateDataDecls(io::Printer* printer) { } auto field_num_to_entry_table = MakeNumToEntryTable(ordered_fields_); format( + "friend class ::$proto_ns$::internal::TcParser;\n" "static const ::$proto_ns$::internal::" "TcParseTable<$1$, $2$, $3$, $4$, $5$> _table_;\n", tc_table_info_->table_size_log2, ordered_fields_.size(), @@ -761,7 +781,7 @@ void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) { // unknown fields and potentially an extension range. auto field_num_to_entry_table = MakeNumToEntryTable(ordered_fields_); format( - "PROTOBUF_ATTRIBUTE_INIT_PRIORITY1\n" + "PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1\n" "const ::_pbi::TcParseTable<$1$, $2$, $3$, $4$, $5$> " "$classname$::_table_ = " "{\n", @@ -1012,6 +1032,11 @@ static void FormatFieldKind(Formatter& format, case FieldDescriptor::TYPE_GROUP: format("Message | ::_fl::kRepGroup"); + if (UseDirectTcParserTable(field, options)) { + format(" | ::_fl::kTvTable"); + } else { + format(" | ::_fl::kTvDefault"); + } break; case FieldDescriptor::TYPE_MESSAGE: if (field->is_map()) { @@ -1023,6 +1048,11 @@ static void FormatFieldKind(Formatter& format, } else if (IsImplicitWeakField(field, options, scc_analyzer)) { format(" | ::_fl::kRepIWeak"); } + if (UseDirectTcParserTable(field, options)) { + format(" | ::_fl::kTvTable"); + } else { + format(" | ::_fl::kTvDefault"); + } } break; } @@ -1052,11 +1082,22 @@ void ParseFunctionGenerator::GenerateFieldEntries(Formatter& format) { } else { const OneofDescriptor* oneof = field->real_containing_oneof(); bool cold = ShouldSplit(field, options_); - format("PROTOBUF_FIELD_OFFSET($classname$$1$, $2$), $3$, $4$,\n ", + format("PROTOBUF_FIELD_OFFSET($classname$$1$, $2$), ", cold ? "::Impl_::Split" : "", cold ? FieldName(field) + "_" - : FieldMemberName(field, /*cold=*/false), - (oneof ? oneof->index() : entry.hasbit_idx), entry.aux_idx); + : FieldMemberName(field, /*cold=*/false)); + if (oneof) { + format("$1$, ", oneof->index()); + } else if (num_hasbits_ > 0 || IsMapEntryMessage(descriptor_)) { + if (entry.hasbit_idx >= 0) { + format("_Internal::kHasBitsOffset + $1$, ", entry.hasbit_idx); + } else { + format("$1$, ", entry.hasbit_idx); + } + } else { + format("0, "); + } + format("$1$,\n ", entry.aux_idx); FormatFieldKind(format, entry, options_, scc_analyzer_); } format("},\n"); @@ -1691,10 +1732,10 @@ std::string FieldParseFunctionName( break; case FieldDescriptor::TYPE_MESSAGE: - name.append("M"); + name.append(UseDirectTcParserTable(field, options) ? "Mt" : "Md"); break; case FieldDescriptor::TYPE_GROUP: - name.append("G"); + name.append(UseDirectTcParserTable(field, options) ? "Gt" : "Gd"); break; default: diff --git a/src/google/protobuf/compiler/cpp/test_bad_identifiers.proto b/src/google/protobuf/compiler/cpp/test_bad_identifiers.proto index 466a84194..028248e61 100644 --- a/src/google/protobuf/compiler/cpp/test_bad_identifiers.proto +++ b/src/google/protobuf/compiler/cpp/test_bad_identifiers.proto @@ -61,7 +61,9 @@ message TestConflictingSymbolNames { optional int32 total_size = 6; optional int32 tag = 7; - enum TestEnum { FOO = 0; } + enum TestEnum { + FOO = 0; + } message Data1 { repeated int32 data = 1; } diff --git a/src/google/protobuf/compiler/cpp/unittest.inc b/src/google/protobuf/compiler/cpp/unittest.inc index 0b4717611..0f841a458 100644 --- a/src/google/protobuf/compiler/cpp/unittest.inc +++ b/src/google/protobuf/compiler/cpp/unittest.inc @@ -56,26 +56,25 @@ // visual studio to compile (report internal errors). #include #endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include +#include +#include +#include #include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include // Must be included last. diff --git a/src/google/protobuf/compiler/csharp/BUILD.bazel b/src/google/protobuf/compiler/csharp/BUILD.bazel new file mode 100644 index 000000000..1271b94ff --- /dev/null +++ b/src/google/protobuf/compiler/csharp/BUILD.bazel @@ -0,0 +1,106 @@ +################################################################################ +# Protocol Buffers Compiler - C# code generator +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("//build_defs:cpp_opts.bzl", "COPTS") + +cc_library( + name = "csharp", + srcs = [ + "csharp_doc_comment.cc", + "csharp_enum.cc", + "csharp_enum_field.cc", + "csharp_field_base.cc", + "csharp_generator.cc", + "csharp_helpers.cc", + "csharp_map_field.cc", + "csharp_message.cc", + "csharp_message_field.cc", + "csharp_primitive_field.cc", + "csharp_reflection_class.cc", + "csharp_repeated_enum_field.cc", + "csharp_repeated_message_field.cc", + "csharp_repeated_primitive_field.cc", + "csharp_source_generator_base.cc", + "csharp_wrapper_field.cc", + ], + hdrs = [ + "csharp_doc_comment.h", + "csharp_enum.h", + "csharp_enum_field.h", + "csharp_field_base.h", + "csharp_generator.h", + "csharp_helpers.h", + "csharp_map_field.h", + "csharp_message.h", + "csharp_message_field.h", + "csharp_names.h", + "csharp_options.h", + "csharp_primitive_field.h", + "csharp_reflection_class.h", + "csharp_repeated_enum_field.h", + "csharp_repeated_message_field.h", + "csharp_repeated_primitive_field.h", + "csharp_source_generator_base.h", + "csharp_wrapper_field.h", + ], + copts = COPTS + select({ + "//build_defs:config_msvc": [], + "//conditions:default": ["-Wno-overloaded-virtual"], + }), + include_prefix = "google/protobuf/compiler/csharp", + visibility = ["//src/google/protobuf/compiler:__pkg__"], + deps = [ + "//:protobuf", + "//src/google/protobuf/compiler:code_generator", + ], +) + +cc_test( + name = "bootstrap_unittest", + srcs = ["csharp_bootstrap_unittest.cc"], + data = [ + "//src/google/protobuf:descriptor_proto_srcs", + "//:well_known_type_protos", + "//conformance:all_files", + "//conformance:conformance_proto", + "//csharp:wkt_cs_srcs", + "//src/google/protobuf:testdata", + ], + deps = [ + ":csharp", + "//:protobuf", + "//src/google/protobuf/compiler:importer", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "generator_unittest", + srcs = ["csharp_generator_unittest.cc"], + deps = [ + ":csharp", + "//:protobuf", + "//src/google/protobuf/compiler:command_line_interface", + "//src/google/protobuf/io", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/compiler/csharp/csharp_reflection_class.cc b/src/google/protobuf/compiler/csharp/csharp_reflection_class.cc index 644fbf16f..d9ab4b49b 100644 --- a/src/google/protobuf/compiler/csharp/csharp_reflection_class.cc +++ b/src/google/protobuf/compiler/csharp/csharp_reflection_class.cc @@ -128,7 +128,7 @@ void ReflectionClassGenerator::WriteIntroduction(io::Printer* printer) { "// Generated by the protocol buffer compiler. DO NOT EDIT!\n" "// source: $file_name$\n" "// \n" - "#pragma warning disable 1591, 0612, 3021\n" + "#pragma warning disable 1591, 0612, 3021, 8981\n" "#region Designer generated code\n" "\n" "using pb = global::Google.Protobuf;\n" diff --git a/src/google/protobuf/compiler/importer.cc b/src/google/protobuf/compiler/importer.cc index f1e26f8bd..595bcd127 100644 --- a/src/google/protobuf/compiler/importer.cc +++ b/src/google/protobuf/compiler/importer.cc @@ -105,6 +105,12 @@ class SourceTreeDescriptorDatabase::SingleFileErrorCollector had_errors_ = true; } + void AddWarning(int line, int column, const std::string& message) override { + if (multi_file_error_collector_ != NULL) { + multi_file_error_collector_->AddWarning(filename_, line, column, message); + } + } + private: std::string filename_; MultiFileErrorCollector* multi_file_error_collector_; diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel new file mode 100644 index 000000000..f035f32de --- /dev/null +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -0,0 +1,123 @@ +################################################################################ +# Protocol Buffers Compiler - Java code generator +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("//build_defs:cpp_opts.bzl", "COPTS") + +cc_library( + name = "java", + srcs = [ + "context.cc", + "doc_comment.cc", + "enum.cc", + "enum_field.cc", + "enum_field_lite.cc", + "enum_lite.cc", + "extension.cc", + "extension_lite.cc", + "field.cc", + "file.cc", + "generator.cc", + "generator_factory.cc", + "helpers.cc", + "kotlin_generator.cc", + "map_field.cc", + "map_field_lite.cc", + "message.cc", + "message_builder.cc", + "message_builder_lite.cc", + "message_field.cc", + "message_field_lite.cc", + "message_lite.cc", + "name_resolver.cc", + "primitive_field.cc", + "primitive_field_lite.cc", + "service.cc", + "shared_code_generator.cc", + "string_field.cc", + "string_field_lite.cc", + ], + hdrs = [ + "context.h", + "doc_comment.h", + "enum.h", + "enum_field.h", + "enum_field_lite.h", + "enum_lite.h", + "extension.h", + "extension_lite.h", + "field.h", + "file.h", + "generator.h", + "generator_factory.h", + "helpers.h", + "kotlin_generator.h", + "map_field.h", + "map_field_lite.h", + "message.h", + "message_builder.h", + "message_builder_lite.h", + "message_field.h", + "message_field_lite.h", + "message_lite.h", + "name_resolver.h", + "names.h", + "options.h", + "primitive_field.h", + "primitive_field_lite.h", + "service.h", + "shared_code_generator.h", + "string_field.h", + "string_field_lite.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/compiler/java", + visibility = ["//src/google/protobuf/compiler:__pkg__"], + deps = [ + "//:protobuf", + "//src/google/protobuf/compiler:code_generator", + ], +) + +cc_test( + name = "doc_comment_unittest", + srcs = ["doc_comment_unittest.cc"], + data = [ + "//src/google/protobuf:descriptor_proto_srcs", + "//:well_known_type_protos", + "//conformance:conformance_proto", + ], + deps = [ + ":java", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "plugin_unittest", + srcs = ["plugin_unittest.cc"], + deps = [ + ":java", + "//:protobuf", + "//src/google/protobuf/compiler:command_line_interface", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs:lite", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/compiler/java/enum_field.cc b/src/google/protobuf/compiler/java/enum_field.cc index ed3594ce7..4a93addbe 100644 --- a/src/google/protobuf/compiler/java/enum_field.cc +++ b/src/google/protobuf/compiler/java/enum_field.cc @@ -42,7 +42,6 @@ #include #include #include -#include #include #include #include @@ -85,13 +84,6 @@ void SetEnumVariables(const FieldDescriptor* descriptor, int messageBitIndex, ? "@kotlin.Deprecated(message = \"Field " + (*variables)["name"] + " is deprecated\") " : ""; - (*variables)["on_changed"] = "onChanged();"; - // Use deprecated valueOf() method to be compatible with old generated code - // for v2.5.0/v2.6.1. - // TODO(xiaofeng): Use "forNumber" when we no longer support compatibility - // with v2.5.0/v2.6.1, and remove the @SuppressWarnings annotations. - (*variables)["for_number"] = "valueOf"; - if (HasHasbit(descriptor)) { // For singular messages and builders, one bit is used for the hasField bit. (*variables)["get_has_field_bit_message"] = GenerateGetBit(messageBitIndex); @@ -204,8 +196,7 @@ void ImmutableEnumFieldGenerator::GenerateMembers(io::Printer* printer) const { printer->Print(variables_, "@java.lang.Override $deprecation$public $type$ " "${$get$capitalized_name$$}$() {\n" - " @SuppressWarnings(\"deprecation\")\n" - " $type$ result = $type$.$for_number$($name$_);\n" + " $type$ result = $type$.forNumber($name$_);\n" " return result == null ? $unknown$ : result;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -238,7 +229,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers( "${$set$capitalized_name$Value$}$(int value) {\n" " $set_has_field_bit_builder$\n" " $name$_ = value;\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -247,8 +238,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers( printer->Print(variables_, "@java.lang.Override\n" "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" - " @SuppressWarnings(\"deprecation\")\n" - " $type$ result = $type$.$for_number$($name$_);\n" + " $type$ result = $type$.forNumber($name$_);\n" " return result == null ? $unknown$ : result;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -262,7 +252,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers( " }\n" " $set_has_field_bit_builder$\n" " $name$_ = value.getNumber();\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -273,7 +263,7 @@ void ImmutableEnumFieldGenerator::GenerateBuilderMembers( "$deprecation$public Builder ${$clear$capitalized_name$$}$() {\n" " $clear_has_field_bit_builder$\n" " $name$_ = $default_number$;\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -364,8 +354,7 @@ void ImmutableEnumFieldGenerator::GenerateParsingCode( } else { printer->Print(variables_, "int rawValue = input.readEnum();\n" - " @SuppressWarnings(\"deprecation\")\n" - "$type$ value = $type$.$for_number$(rawValue);\n" + "$type$ value = $type$.forNumber(rawValue);\n" "if (value == null) {\n" " unknownFields.mergeVarintField($number$, rawValue);\n" "} else {\n" @@ -453,8 +442,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateMembers( printer->Print(variables_, "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" " if ($has_oneof_case_message$) {\n" - " @SuppressWarnings(\"deprecation\")\n" - " $type$ result = $type$.$for_number$(\n" + " $type$ result = $type$.forNumber(\n" " (java.lang.Integer) $oneof_name$_);\n" " return result == null ? $unknown$ : result;\n" " }\n" @@ -493,7 +481,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers( "${$set$capitalized_name$Value$}$(int value) {\n" " $set_oneof_case_message$;\n" " $oneof_name$_ = value;\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -503,8 +491,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers( "@java.lang.Override\n" "$deprecation$public $type$ ${$get$capitalized_name$$}$() {\n" " if ($has_oneof_case_message$) {\n" - " @SuppressWarnings(\"deprecation\")\n" - " $type$ result = $type$.$for_number$(\n" + " $type$ result = $type$.forNumber(\n" " (java.lang.Integer) $oneof_name$_);\n" " return result == null ? $unknown$ : result;\n" " }\n" @@ -521,7 +508,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers( " }\n" " $set_oneof_case_message$;\n" " $oneof_name$_ = value.getNumber();\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -533,7 +520,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateBuilderMembers( " if ($has_oneof_case_message$) {\n" " $clear_oneof_case_message$;\n" " $oneof_name$_ = null;\n" - " $on_changed$\n" + " onChanged();\n" " }\n" " return this;\n" "}\n"); @@ -570,8 +557,7 @@ void ImmutableEnumOneofFieldGenerator::GenerateParsingCode( } else { printer->Print(variables_, "int rawValue = input.readEnum();\n" - "@SuppressWarnings(\"deprecation\")\n" - "$type$ value = $type$.$for_number$(rawValue);\n" + "$type$ value = $type$.forNumber(rawValue);\n" "if (value == null) {\n" " unknownFields.mergeVarintField($number$, rawValue);\n" "} else {\n" @@ -685,8 +671,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateMembers( " new com.google.protobuf.Internal.ListAdapter.Converter<\n" " java.lang.Integer, $type$>() {\n" " public $type$ convert(java.lang.Integer from) {\n" - " @SuppressWarnings(\"deprecation\")\n" - " $type$ result = $type$.$for_number$(from);\n" + " $type$ result = $type$.forNumber(from);\n" " return result == null ? $unknown$ : result;\n" " }\n" " };\n"); @@ -802,7 +787,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " }\n" " ensure$capitalized_name$IsMutable();\n" " $name$_.set(index, value.getNumber());\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -816,7 +801,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " }\n" " ensure$capitalized_name$IsMutable();\n" " $name$_.add(value.getNumber());\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -829,7 +814,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " for ($type$ value : values) {\n" " $name$_.add(value.getNumber());\n" " }\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -840,7 +825,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( "$deprecation$public Builder ${$clear$capitalized_name$$}$() {\n" " $name$_ = java.util.Collections.emptyList();\n" " $clear_mutable_bit_builder$;\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -870,7 +855,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " int index, int value) {\n" " ensure$capitalized_name$IsMutable();\n" " $name$_.set(index, value);\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -881,7 +866,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( "${$add$capitalized_name$Value$}$(int value) {\n" " ensure$capitalized_name$IsMutable();\n" " $name$_.add(value);\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -895,7 +880,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateBuilderMembers( " for (int value : values) {\n" " $name$_.add(value);\n" " }\n" - " $on_changed$\n" + " onChanged();\n" " return this;\n" "}\n"); printer->Annotate("{", "}", descriptor_); @@ -935,7 +920,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateMergingCode( " ensure$capitalized_name$IsMutable();\n" " $name$_.addAll(other.$name$_);\n" " }\n" - " $on_changed$\n" + " onChanged();\n" "}\n"); } @@ -967,8 +952,7 @@ void RepeatedImmutableEnumFieldGenerator::GenerateParsingCode( printer->Print( variables_, "int rawValue = input.readEnum();\n" - "@SuppressWarnings(\"deprecation\")\n" - "$type$ value = $type$.$for_number$(rawValue);\n" + "$type$ value = $type$.forNumber(rawValue);\n" "if (value == null) {\n" " unknownFields.mergeVarintField($number$, rawValue);\n" "} else {\n" diff --git a/src/google/protobuf/compiler/java/map_field.cc b/src/google/protobuf/compiler/java/map_field.cc index bc48f61a7..ce0766129 100644 --- a/src/google/protobuf/compiler/java/map_field.cc +++ b/src/google/protobuf/compiler/java/map_field.cc @@ -127,6 +127,8 @@ void SetMessageVariables(const FieldDescriptor* descriptor, int messageBitIndex, if (valueJavaType == JAVATYPE_ENUM) { // We store enums as Integers internally. (*variables)["value_type"] = "int"; + (*variables)["value_type_pass_through_nullness"] = + (*variables)["value_type"]; (*variables)["boxed_value_type"] = "java.lang.Integer"; (*variables)["value_wire_type"] = WireType(value); (*variables)["value_default_value"] = @@ -262,9 +264,10 @@ void ImmutableMapFieldGenerator::GenerateInterfaceMembers( WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, "$deprecation$\n" - "$value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" + "$value_type_pass_through_nullness$ " + "${$get$capitalized_name$ValueOrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue);\n"); + " $value_type_pass_through_nullness$ defaultValue);\n"); printer->Annotate("{", "}", descriptor_); WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, @@ -609,9 +612,10 @@ void ImmutableMapFieldGenerator::GenerateMapGetters( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$ValueOrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " internalGet$capitalized_name$().getMap();\n" @@ -661,9 +665,10 @@ void ImmutableMapFieldGenerator::GenerateMapGetters( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$OrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$OrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$type_parameters$> map =\n" " internalGet$capitalized_name$().getMap();\n" diff --git a/src/google/protobuf/compiler/java/map_field_lite.cc b/src/google/protobuf/compiler/java/map_field_lite.cc index 4f2766e8f..7e5f829ba 100644 --- a/src/google/protobuf/compiler/java/map_field_lite.cc +++ b/src/google/protobuf/compiler/java/map_field_lite.cc @@ -126,6 +126,8 @@ void SetMessageVariables(const FieldDescriptor* descriptor, int messageBitIndex, if (GetJavaType(value) == JAVATYPE_ENUM) { // We store enums as Integers internally. (*variables)["value_type"] = "int"; + (*variables)["value_type_pass_through_nullness"] = + (*variables)["value_type"]; (*variables)["boxed_value_type"] = "java.lang.Integer"; (*variables)["value_wire_type"] = WireType(value); (*variables)["value_default_value"] = @@ -247,9 +249,10 @@ void ImmutableMapFieldLiteGenerator::GenerateInterfaceMembers( WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, "$deprecation$\n" - "$value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" + "$value_type_pass_through_nullness$ " + "${$get$capitalized_name$ValueOrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue);\n"); + " $value_type_pass_through_nullness$ defaultValue);\n"); printer->Annotate("{", "}", descriptor_); WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, @@ -373,9 +376,10 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_enum_type$ ${$get$capitalized_name$OrDefault$}$(\n" + "public $value_enum_type_pass_through_nullness$ " + "${$get$capitalized_name$OrDefault$}$(\n" " $key_type$ key,\n" - " $value_enum_type$ defaultValue) {\n" + " $value_enum_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " internalGet$capitalized_name$();\n" @@ -429,9 +433,10 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$ValueOrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " internalGet$capitalized_name$();\n" @@ -482,9 +487,10 @@ void ImmutableMapFieldLiteGenerator::GenerateMembers( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$OrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$OrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$type_parameters$> map =\n" " internalGet$capitalized_name$();\n" @@ -701,9 +707,10 @@ void ImmutableMapFieldLiteGenerator::GenerateBuilderMembers( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$ValueOrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$ValueOrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$boxed_key_type$, $boxed_value_type$> map =\n" " instance.get$capitalized_name$ValueMap();\n" @@ -776,9 +783,10 @@ void ImmutableMapFieldLiteGenerator::GenerateBuilderMembers( variables_, "@java.lang.Override\n" "$deprecation$\n" - "public $value_type$ ${$get$capitalized_name$OrDefault$}$(\n" + "public $value_type_pass_through_nullness$ " + "${$get$capitalized_name$OrDefault$}$(\n" " $key_type$ key,\n" - " $value_type$ defaultValue) {\n" + " $value_type_pass_through_nullness$ defaultValue) {\n" " $key_null_check$\n" " java.util.Map<$type_parameters$> map =\n" " instance.get$capitalized_name$Map();\n" diff --git a/src/google/protobuf/compiler/java/message.cc b/src/google/protobuf/compiler/java/message.cc index 0cc617173..95917a7f6 100644 --- a/src/google/protobuf/compiler/java/message.cc +++ b/src/google/protobuf/compiler/java/message.cc @@ -425,6 +425,8 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) { vars["oneof_capitalized_name"] = context_->GetOneofGeneratorInfo(oneof)->capitalized_name; vars["oneof_index"] = StrCat((oneof)->index()); + vars["{"] = ""; + vars["}"] = ""; // oneofCase_ and oneof_ printer->Print(vars, "private int $oneof_name$Case_ = 0;\n" @@ -432,11 +434,12 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) { // OneofCase enum printer->Print( vars, - "public enum $oneof_capitalized_name$Case\n" + "public enum ${$$oneof_capitalized_name$Case$}$\n" // TODO(dweis): Remove EnumLite when we want to break compatibility with // 3.x users " implements com.google.protobuf.Internal.EnumLite,\n" " com.google.protobuf.AbstractMessage.InternalOneOfEnum {\n"); + printer->Annotate("{", "}", oneof); printer->Indent(); for (int j = 0; j < (oneof)->field_count(); j++) { const FieldDescriptor* field = (oneof)->field(j); @@ -445,6 +448,7 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) { field->options().deprecated() ? "@java.lang.Deprecated " : "", "field_name", ToUpper(field->name()), "field_number", StrCat(field->number())); + printer->Annotate("field_name", field); } printer->Print("$cap_oneof_name$_NOT_SET(0);\n", "cap_oneof_name", ToUpper(vars["oneof_name"])); @@ -487,11 +491,12 @@ void ImmutableMessageGenerator::Generate(io::Printer* printer) { // oneofCase() printer->Print(vars, "public $oneof_capitalized_name$Case\n" - "get$oneof_capitalized_name$Case() {\n" + "${$get$oneof_capitalized_name$Case$}$() {\n" " return $oneof_capitalized_name$Case.forNumber(\n" " $oneof_name$Case_);\n" "}\n" "\n"); + printer->Annotate("{", "}", oneof); } if (IsAnyMessage(descriptor_)) { @@ -1012,6 +1017,8 @@ void ImmutableMessageGenerator::GenerateEqualsAndHashCode( " return true;\n" "}\n" "if (!(obj instanceof $classname$)) {\n" + // don't simply return false because mutable and immutable types + // can be equal " return super.equals(obj);\n" "}\n" "$classname$ other = ($classname$) obj;\n" diff --git a/src/google/protobuf/compiler/objectivec/BUILD.bazel b/src/google/protobuf/compiler/objectivec/BUILD.bazel new file mode 100644 index 000000000..071539075 --- /dev/null +++ b/src/google/protobuf/compiler/objectivec/BUILD.bazel @@ -0,0 +1,69 @@ +################################################################################ +# Protocol Buffers Compiler - ObjC code generator +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("//build_defs:cpp_opts.bzl", "COPTS") + +cc_library( + name = "objectivec", + srcs = [ + "objectivec_enum.cc", + "objectivec_enum_field.cc", + "objectivec_extension.cc", + "objectivec_field.cc", + "objectivec_file.cc", + "objectivec_generator.cc", + "objectivec_helpers.cc", + "objectivec_map_field.cc", + "objectivec_message.cc", + "objectivec_message_field.cc", + "objectivec_oneof.cc", + "objectivec_primitive_field.cc", + ], + hdrs = [ + "objectivec_enum.h", + "objectivec_enum_field.h", + "objectivec_extension.h", + "objectivec_field.h", + "objectivec_file.h", + "objectivec_generator.h", + "objectivec_helpers.h", + "objectivec_map_field.h", + "objectivec_message.h", + "objectivec_message_field.h", + "objectivec_nsobject_methods.h", + "objectivec_oneof.h", + "objectivec_primitive_field.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/compiler/objectivec", + visibility = ["//src/google/protobuf/compiler:__pkg__"], + deps = [ + "//:protobuf", + "//src/google/protobuf/compiler:code_generator", + ], +) + +cc_test( + name = "helpers_unittest", + srcs = ["objectivec_helpers_unittest.cc"], + deps = [ + ":objectivec", + "//src/google/protobuf/io", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/compiler/parser.cc b/src/google/protobuf/compiler/parser.cc index 5bd37d147..472394dbb 100644 --- a/src/google/protobuf/compiler/parser.cc +++ b/src/google/protobuf/compiler/parser.cc @@ -643,10 +643,11 @@ bool Parser::Parse(io::Tokenizer* input, FileDescriptorProto* file) { // Store the syntax into the file. if (file != nullptr) file->set_syntax(syntax_identifier_); } else if (!stop_after_syntax_identifier_) { - GOOGLE_LOG(WARNING) << "No syntax specified for the proto file: " << file->name() - << ". Please use 'syntax = \"proto2\";' " - << "or 'syntax = \"proto3\";' to specify a syntax " - << "version. (Defaulted to proto2 syntax.)"; + AddWarning( + "No syntax specified. Please use 'syntax = \"proto2\";' or " + "'syntax = \"proto3\";' to specify a syntax version. " + "(Defaulted to proto2 syntax.)" + ); syntax_identifier_ = "proto2"; } @@ -1019,7 +1020,8 @@ bool Parser::ParseMessageFieldNoLabel( location.RecordLegacyLocation(field, DescriptorPool::ErrorCollector::NAME); DO(ConsumeIdentifier(field->mutable_name(), "Expected field name.")); - if (!IsLowerUnderscore(field->name())) { + if (field->type() != FieldDescriptorProto::TYPE_GROUP && + !IsLowerUnderscore(field->name())) { AddWarning( "Field name should be lowercase. Found: " + field->name() + ". See: https://developers.google.com/protocol-buffers/docs/style"); diff --git a/src/google/protobuf/compiler/parser_unittest.cc b/src/google/protobuf/compiler/parser_unittest.cc index 2d681d957..83f591dc3 100644 --- a/src/google/protobuf/compiler/parser_unittest.cc +++ b/src/google/protobuf/compiler/parser_unittest.cc @@ -221,9 +221,8 @@ TEST_F(ParserTest, StopAfterSyntaxIdentifierWithErrors) { TEST_F(ParserTest, WarnIfSyntaxIdentifierOmmitted) { SetupParser("message A {}"); FileDescriptorProto file; - CaptureTestStderr(); EXPECT_TRUE(parser_->Parse(input_.get(), &file)); - EXPECT_TRUE(GetCapturedTestStderr().find("No syntax specified") != + EXPECT_TRUE(error_collector_.warning_.find("No syntax specified") != std::string::npos); } diff --git a/src/google/protobuf/compiler/php/BUILD.bazel b/src/google/protobuf/compiler/php/BUILD.bazel new file mode 100644 index 000000000..969d2df84 --- /dev/null +++ b/src/google/protobuf/compiler/php/BUILD.bazel @@ -0,0 +1,31 @@ +################################################################################ +# Protocol Buffers Compiler - PHP code generator +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("//build_defs:cpp_opts.bzl", "COPTS") + +cc_library( + name = "php", + srcs = ["php_generator.cc"], + hdrs = ["php_generator.h"], + copts = COPTS, + include_prefix = "google/protobuf/compiler/php", + visibility = ["//src/google/protobuf/compiler:__pkg__"], + deps = [ + "//:protobuf", + "//src/google/protobuf/compiler:code_generator", + ], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/compiler/php/php_generator.cc b/src/google/protobuf/compiler/php/php_generator.cc index 135a92f65..f181a39ef 100644 --- a/src/google/protobuf/compiler/php/php_generator.cc +++ b/src/google/protobuf/compiler/php/php_generator.cc @@ -407,19 +407,6 @@ std::string GeneratedClassFileName(const DescriptorType* desc, return result + ".php"; } -template -std::string LegacyGeneratedClassFileName(const DescriptorType* desc, - const Options& options) { - std::string result = LegacyFullClassName(desc, options); - - for (int i = 0; i < result.size(); i++) { - if (result[i] == '\\') { - result[i] = '/'; - } - } - return result + ".php"; -} - template std::string LegacyReadOnlyGeneratedClassFileName(const DescriptorType* desc, const Options& options) { @@ -1275,43 +1262,6 @@ void GenerateMetadataFile(const FileDescriptor* file, const Options& options, printer.Print("}\n\n"); } -template -void LegacyGenerateClassFile(const FileDescriptor* file, - const DescriptorType* desc, const Options& options, - GeneratorContext* generator_context) { - std::string filename = LegacyGeneratedClassFileName(desc, options); - std::unique_ptr output( - generator_context->Open(filename)); - io::Printer printer(output.get(), '^'); - - GenerateHead(file, &printer); - - std::string php_namespace = RootPhpNamespace(desc, options); - if (!php_namespace.empty()) { - printer.Print( - "namespace ^name^;\n\n", - "name", php_namespace); - } - std::string newname = FullClassName(desc, options); - printer.Print("if (false) {\n"); - Indent(&printer); - printer.Print("/**\n"); - printer.Print(" * This class is deprecated. Use ^new^ instead.\n", - "new", newname); - printer.Print(" * @deprecated\n"); - printer.Print(" */\n"); - printer.Print("class ^old^ {}\n", - "old", LegacyGeneratedClassName(desc)); - Outdent(&printer); - printer.Print("}\n"); - printer.Print("class_exists(^new^::class);\n", - "new", GeneratedClassNameImpl(desc)); - printer.Print("@trigger_error('^old^ is deprecated and will be removed in " - "the next major release. Use ^fullname^ instead', E_USER_DEPRECATED);\n\n", - "old", LegacyFullClassName(desc, options), - "fullname", newname); -} - template void LegacyReadOnlyGenerateClassFile(const FileDescriptor* file, const DescriptorType* desc, const Options& options, @@ -1449,7 +1399,7 @@ void GenerateEnumFile(const FileDescriptor* file, const EnumDescriptor* en, Outdent(&printer); printer.Print("}\n\n"); - // write legacy file for backwards compatibility with nested messages and enums + // write legacy alias for backwards compatibility with nested messages and enums if (en->containing_type() != NULL) { printer.Print( "// Adding a class alias for backwards compatibility with the previous class name.\n"); @@ -1457,7 +1407,6 @@ void GenerateEnumFile(const FileDescriptor* file, const EnumDescriptor* en, "class_alias(^new^::class, \\^old^::class);\n\n", "new", fullname, "old", LegacyFullClassName(en, options)); - LegacyGenerateClassFile(file, en, options, generator_context); } // Write legacy file for backwards compatibility with "readonly" keywword @@ -1577,7 +1526,7 @@ void GenerateMessageFile(const FileDescriptor* file, const Descriptor* message, Outdent(&printer); printer.Print("}\n\n"); - // write legacy file for backwards compatibility with nested messages and enums + // write legacy alias for backwards compatibility with nested messages and enums if (message->containing_type() != NULL) { printer.Print( "// Adding a class alias for backwards compatibility with the previous class name.\n"); @@ -1585,7 +1534,6 @@ void GenerateMessageFile(const FileDescriptor* file, const Descriptor* message, "class_alias(^new^::class, \\^old^::class);\n\n", "new", fullname, "old", LegacyFullClassName(message, options)); - LegacyGenerateClassFile(file, message, options, generator_context); } // Write legacy file for backwards compatibility with "readonly" keywword diff --git a/src/google/protobuf/compiler/plugin.pb.cc b/src/google/protobuf/compiler/plugin.pb.cc index 8f8d83f02..def353e21 100644 --- a/src/google/protobuf/compiler/plugin.pb.cc +++ b/src/google/protobuf/compiler/plugin.pb.cc @@ -234,6 +234,8 @@ constexpr int CodeGeneratorResponse::Feature_ARRAYSIZE; class Version::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(Version, _impl_._has_bits_); static void set_has_major(HasBits* has_bits) { (*has_bits)[0] |= 2u; } @@ -558,6 +560,8 @@ void Version::InternalSwap(Version* other) { class CodeGeneratorRequest::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(CodeGeneratorRequest, _impl_._has_bits_); static void set_has_parameter(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -910,6 +914,8 @@ void CodeGeneratorRequest::InternalSwap(CodeGeneratorRequest* other) { class CodeGeneratorResponse_File::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(CodeGeneratorResponse_File, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -1300,6 +1306,8 @@ void CodeGeneratorResponse_File::InternalSwap(CodeGeneratorResponse_File* other) class CodeGeneratorResponse::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(CodeGeneratorResponse, _impl_._has_bits_); static void set_has_error(HasBits* has_bits) { (*has_bits)[0] |= 1u; } diff --git a/src/google/protobuf/compiler/plugin.pb.h b/src/google/protobuf/compiler/plugin.pb.h index 2bbe3b5b6..b8829b495 100644 --- a/src/google/protobuf/compiler/plugin.pb.h +++ b/src/google/protobuf/compiler/plugin.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/compiler/python/BUILD.bazel b/src/google/protobuf/compiler/python/BUILD.bazel new file mode 100644 index 000000000..3fe6807d1 --- /dev/null +++ b/src/google/protobuf/compiler/python/BUILD.bazel @@ -0,0 +1,53 @@ +################################################################################ +# Protocol Buffers Compiler - Python code generator +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("//build_defs:cpp_opts.bzl", "COPTS") + +cc_library( + name = "python", + srcs = [ + "generator.cc", + "helpers.cc", + "pyi_generator.cc", + ], + hdrs = [ + "generator.h", + "helpers.h", + "pyi_generator.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/compiler/python", + visibility = ["//src/google/protobuf/compiler:__pkg__"], + deps = [ + "//:protobuf", + "//src/google/protobuf/compiler:code_generator", + ], +) + +cc_test( + name = "plugin_unittest", + srcs = ["plugin_unittest.cc"], + copts = COPTS, + deps = [ + ":python", + "//src/google/protobuf/compiler:command_line_interface", + "//src/google/protobuf/io", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/compiler/ruby/BUILD.bazel b/src/google/protobuf/compiler/ruby/BUILD.bazel new file mode 100644 index 000000000..3a3469eb0 --- /dev/null +++ b/src/google/protobuf/compiler/ruby/BUILD.bazel @@ -0,0 +1,58 @@ +################################################################################ +# Protocol Buffers Compiler - Ruby code generator +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("//build_defs:cpp_opts.bzl", "COPTS") + +cc_library( + name = "ruby", + srcs = ["ruby_generator.cc"], + hdrs = ["ruby_generator.h"], + copts = COPTS, + include_prefix = "google/protobuf/compiler/ruby", + visibility = ["//src/google/protobuf/compiler:__pkg__"], + deps = [ + "//:protobuf", + "//src/google/protobuf/compiler:code_generator", + ], +) + +cc_test( + name = "generator_unittest", + srcs = ["ruby_generator_unittest.cc"], + data = [ + "ruby_generated_code.proto", + "ruby_generated_code_pb.rb", + "ruby_generated_code_proto2.proto", + "ruby_generated_code_proto2_import.proto", + "ruby_generated_code_proto2_pb.rb", + "ruby_generated_pkg_explicit.proto", + "ruby_generated_pkg_explicit_legacy.proto", + "ruby_generated_pkg_explicit_legacy_pb.rb", + "ruby_generated_pkg_explicit_pb.rb", + "ruby_generated_pkg_implicit.proto", + "ruby_generated_pkg_implicit_pb.rb", + "//src/google/protobuf:testdata", + ], + deps = [ + ":ruby", + "//src/google/protobuf/compiler:command_line_interface", + "//src/google/protobuf/io", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index d3bfb461e..645096686 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -1512,6 +1512,8 @@ void FileDescriptorSet::InternalSwap(FileDescriptorSet* other) { class FileDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(FileDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -2209,6 +2211,8 @@ void FileDescriptorProto::InternalSwap(FileDescriptorProto* other) { class DescriptorProto_ExtensionRange::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(DescriptorProto_ExtensionRange, _impl_._has_bits_); static void set_has_start(HasBits* has_bits) { (*has_bits)[0] |= 2u; } @@ -2493,6 +2497,8 @@ void DescriptorProto_ExtensionRange::InternalSwap(DescriptorProto_ExtensionRange class DescriptorProto_ReservedRange::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(DescriptorProto_ReservedRange, _impl_._has_bits_); static void set_has_start(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -2730,6 +2736,8 @@ void DescriptorProto_ReservedRange::InternalSwap(DescriptorProto_ReservedRange* class DescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(DescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -3505,6 +3513,8 @@ void ExtensionRangeOptions::InternalSwap(ExtensionRangeOptions* other) { class FieldDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(FieldDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -4186,6 +4196,8 @@ void FieldDescriptorProto::InternalSwap(FieldDescriptorProto* other) { class OneofDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(OneofDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -4461,6 +4473,8 @@ void OneofDescriptorProto::InternalSwap(OneofDescriptorProto* other) { class EnumDescriptorProto_EnumReservedRange::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(EnumDescriptorProto_EnumReservedRange, _impl_._has_bits_); static void set_has_start(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -4698,6 +4712,8 @@ void EnumDescriptorProto_EnumReservedRange::InternalSwap(EnumDescriptorProto_Enu class EnumDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(EnumDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -5084,6 +5100,8 @@ void EnumDescriptorProto::InternalSwap(EnumDescriptorProto* other) { class EnumValueDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(EnumValueDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -5395,6 +5413,8 @@ void EnumValueDescriptorProto::InternalSwap(EnumValueDescriptorProto* other) { class ServiceDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(ServiceDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -5706,6 +5726,8 @@ void ServiceDescriptorProto::InternalSwap(ServiceDescriptorProto* other) { class MethodDescriptorProto::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(MethodDescriptorProto, _impl_._has_bits_); static void set_has_name(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -6163,6 +6185,8 @@ void MethodDescriptorProto::InternalSwap(MethodDescriptorProto* other) { class FileOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(FileOptions, _impl_._has_bits_); static void set_has_java_package(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -7282,6 +7306,8 @@ void FileOptions::InternalSwap(FileOptions* other) { class MessageOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(MessageOptions, _impl_._has_bits_); static void set_has_message_set_wire_format(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -7630,6 +7656,8 @@ void MessageOptions::InternalSwap(MessageOptions* other) { class FieldOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(FieldOptions, _impl_._has_bits_); static void set_has_ctype(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -8286,6 +8314,8 @@ void OneofOptions::InternalSwap(OneofOptions* other) { class EnumOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(EnumOptions, _impl_._has_bits_); static void set_has_allow_alias(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -8578,6 +8608,8 @@ void EnumOptions::InternalSwap(EnumOptions* other) { class EnumValueOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(EnumValueOptions, _impl_._has_bits_); static void set_has_deprecated(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -8827,6 +8859,8 @@ void EnumValueOptions::InternalSwap(EnumValueOptions* other) { class ServiceOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(ServiceOptions, _impl_._has_bits_); static void set_has_deprecated(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -9076,6 +9110,8 @@ void ServiceOptions::InternalSwap(ServiceOptions* other) { class MethodOptions::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(MethodOptions, _impl_._has_bits_); static void set_has_deprecated(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -9377,6 +9413,8 @@ void MethodOptions::InternalSwap(MethodOptions* other) { class UninterpretedOption_NamePart::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(UninterpretedOption_NamePart, _impl_._has_bits_); static void set_has_name_part(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -9653,6 +9691,8 @@ void UninterpretedOption_NamePart::InternalSwap(UninterpretedOption_NamePart* ot class UninterpretedOption::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(UninterpretedOption, _impl_._has_bits_); static void set_has_identifier_value(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -10122,6 +10162,8 @@ void UninterpretedOption::InternalSwap(UninterpretedOption* other) { class SourceCodeInfo_Location::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(SourceCodeInfo_Location, _impl_._has_bits_); static void set_has_leading_comments(HasBits* has_bits) { (*has_bits)[0] |= 1u; } @@ -10716,6 +10758,8 @@ void SourceCodeInfo::InternalSwap(SourceCodeInfo* other) { class GeneratedCodeInfo_Annotation::_Internal { public: using HasBits = decltype(std::declval()._impl_._has_bits_); + static constexpr int32_t kHasBitsOffset = + 8 * PROTOBUF_FIELD_OFFSET(GeneratedCodeInfo_Annotation, _impl_._has_bits_); static void set_has_source_file(HasBits* has_bits) { (*has_bits)[0] |= 1u; } diff --git a/src/google/protobuf/descriptor.pb.h b/src/google/protobuf/descriptor.pb.h index 07ed8f434..0a51ba353 100644 --- a/src/google/protobuf/descriptor.pb.h +++ b/src/google/protobuf/descriptor.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/descriptor.proto b/src/google/protobuf/descriptor.proto index f8eb216cd..5b4d06b41 100644 --- a/src/google/protobuf/descriptor.proto +++ b/src/google/protobuf/descriptor.proto @@ -604,11 +604,8 @@ message FieldOptions { // check its required fields, regardless of whether or not the message has // been parsed. // - // As of 2021, lazy does no correctness checks on the byte stream during - // parsing. This may lead to crashes if and when an invalid byte stream is - // finally parsed upon access. - // - // TODO(b/211906113): Enable validation on lazy fields. + // As of May 2022, lazy verifies the contents of the byte stream during + // parsing. An invalid byte stream will cause the overall parsing to fail. optional bool lazy = 5 [default = false]; // unverified_lazy does no correctness checks on the byte stream. This should diff --git a/src/google/protobuf/duration.pb.h b/src/google/protobuf/duration.pb.h index 555ae47ab..7bdff41c5 100644 --- a/src/google/protobuf/duration.pb.h +++ b/src/google/protobuf/duration.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index 8aab9f59a..70fd45e60 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/field_mask.pb.h b/src/google/protobuf/field_mask.pb.h index 7f47936a9..9022e1f31 100644 --- a/src/google/protobuf/field_mask.pb.h +++ b/src/google/protobuf/field_mask.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 2a807e066..53ca9acd3 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -983,8 +983,6 @@ void Reflection::Swap(Message* message1, Message* message2) const { return; } - GOOGLE_DCHECK_EQ(message1->GetOwningArena(), message2->GetOwningArena()); - UnsafeArenaSwap(message1, message2); } @@ -1014,9 +1012,6 @@ void Reflection::SwapFieldsImpl( std::set swapped_oneof; - GOOGLE_DCHECK(!unsafe_shallow_swap || message1->GetArenaForAllocation() == - message2->GetArenaForAllocation()); - const Message* prototype = message_factory_->GetPrototype(message1->GetDescriptor()); for (const auto* field : fields) { @@ -1073,6 +1068,9 @@ void Reflection::SwapFields( void Reflection::UnsafeShallowSwapFields( Message* message1, Message* message2, const std::vector& fields) const { + GOOGLE_DCHECK_EQ(message1->GetArenaForAllocation(), + message2->GetArenaForAllocation()); + SwapFieldsImpl(message1, message2, fields); } @@ -1103,6 +1101,11 @@ bool Reflection::HasField(const Message& message, } void Reflection::UnsafeArenaSwap(Message* lhs, Message* rhs) const { + GOOGLE_DCHECK_EQ(lhs->GetOwningArena(), rhs->GetOwningArena()); + InternalSwap(lhs, rhs); +} + +void Reflection::InternalSwap(Message* lhs, Message* rhs) const { if (lhs == rhs) return; MutableInternalMetadata(lhs)->InternalSwap(MutableInternalMetadata(rhs)); diff --git a/src/google/protobuf/generated_message_tctable_decl.h b/src/google/protobuf/generated_message_tctable_decl.h index b1bb1def7..3974dc34a 100644 --- a/src/google/protobuf/generated_message_tctable_decl.h +++ b/src/google/protobuf/generated_message_tctable_decl.h @@ -120,6 +120,8 @@ struct Offset { #pragma warning(disable : 4324) #endif +struct FieldAuxDefaultMessage {}; + // Base class for message-level table with info for the tail-call parser. struct alignas(uint64_t) TcParseTableBase { // Common attributes for message layout: @@ -191,7 +193,7 @@ struct alignas(uint64_t) TcParseTableBase { // Field entry for all fields. struct FieldEntry { uint32_t offset; // offset in the message object - int32_t has_idx; // has-bit index + int32_t has_idx; // has-bit index, relative to the message object uint16_t aux_idx; // index for `field_aux`. uint16_t type_card; // `FieldType` and `Cardinality` (see _impl.h) }; @@ -204,20 +206,28 @@ struct alignas(uint64_t) TcParseTableBase { // Auxiliary entries for field types that need extra information. union FieldAux { - constexpr FieldAux() : message_default(nullptr) {} + constexpr FieldAux() : message_default_p(nullptr) {} constexpr FieldAux(bool (*enum_validator)(int)) : enum_validator(enum_validator) {} constexpr FieldAux(field_layout::Offset off) : offset(off.off) {} constexpr FieldAux(int16_t range_start, uint16_t range_length) : enum_range{range_start, range_length} {} - constexpr FieldAux(const MessageLite* msg) : message_default(msg) {} + constexpr FieldAux(const MessageLite* msg) : message_default_p(msg) {} + constexpr FieldAux(FieldAuxDefaultMessage, const void* msg) + : message_default_p(msg) {} + constexpr FieldAux(const TcParseTableBase* table) : table(table) {} bool (*enum_validator)(int); struct { int16_t start; // minimum enum number (if it fits) uint16_t length; // length of range (i.e., max = start + length - 1) } enum_range; uint32_t offset; - const MessageLite* message_default; + const void* message_default_p; + const TcParseTableBase* table; + + const MessageLite* message_default() const { + return static_cast(message_default_p); + } }; const FieldAux* field_aux(uint32_t idx) const { return reinterpret_cast(reinterpret_cast(this) + diff --git a/src/google/protobuf/generated_message_tctable_impl.h b/src/google/protobuf/generated_message_tctable_impl.h index 21fa5332d..c64b264dc 100644 --- a/src/google/protobuf/generated_message_tctable_impl.h +++ b/src/google/protobuf/generated_message_tctable_impl.h @@ -147,6 +147,10 @@ enum TransformValidation : uint16_t { // String fields: kTvUtf8Debug = 1 << kTvShift, // proto2 kTvUtf8 = 2 << kTvShift, // proto3 + + // Message fields: + kTvDefault = 1 << kTvShift, // Aux has default_instance + kTvTable = 2 << kTvShift, // Aux has TcParseTableBase* }; static_assert((kTvEnum & kTvRange) != 0, @@ -262,6 +266,11 @@ extern template void AlignFail<8>(uintptr_t); // TcParser implements most of the parsing logic for tailcall tables. class PROTOBUF_EXPORT TcParser final { public: + template + static constexpr const TcParseTableBase* GetTable() { + return &T::_table_.header; + } + static const char* GenericFallback(PROTOBUF_TC_PARAM_DECL); static const char* GenericFallbackLite(PROTOBUF_TC_PARAM_DECL); @@ -364,16 +373,26 @@ class PROTOBUF_EXPORT TcParser final { // Functions referenced by generated fast tables (message types): // M: message G: group + // d: default* t: TcParseTable* (the contents of aux) // S: singular R: repeated // 1/2: tag length (bytes) - static const char* FastMS1(PROTOBUF_TC_PARAM_DECL); - static const char* FastMS2(PROTOBUF_TC_PARAM_DECL); - static const char* FastMR1(PROTOBUF_TC_PARAM_DECL); - static const char* FastMR2(PROTOBUF_TC_PARAM_DECL); - static const char* FastGS1(PROTOBUF_TC_PARAM_DECL); - static const char* FastGS2(PROTOBUF_TC_PARAM_DECL); - static const char* FastGR1(PROTOBUF_TC_PARAM_DECL); - static const char* FastGR2(PROTOBUF_TC_PARAM_DECL); + static const char* FastMdS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastMdS2(PROTOBUF_TC_PARAM_DECL); + static const char* FastGdS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastGdS2(PROTOBUF_TC_PARAM_DECL); + static const char* FastMtS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastMtS2(PROTOBUF_TC_PARAM_DECL); + static const char* FastGtS1(PROTOBUF_TC_PARAM_DECL); + static const char* FastGtS2(PROTOBUF_TC_PARAM_DECL); + + static const char* FastMdR1(PROTOBUF_TC_PARAM_DECL); + static const char* FastMdR2(PROTOBUF_TC_PARAM_DECL); + static const char* FastGdR1(PROTOBUF_TC_PARAM_DECL); + static const char* FastGdR2(PROTOBUF_TC_PARAM_DECL); + static const char* FastMtR1(PROTOBUF_TC_PARAM_DECL); + static const char* FastMtR2(PROTOBUF_TC_PARAM_DECL); + static const char* FastGtR1(PROTOBUF_TC_PARAM_DECL); + static const char* FastGtR2(PROTOBUF_TC_PARAM_DECL); template static inline T& RefAt(void* x, size_t offset) { @@ -421,9 +440,9 @@ class PROTOBUF_EXPORT TcParser final { private: friend class GeneratedTcTableLiteTest; - template + template static inline const char* SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); - template + template static inline const char* RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL); static inline PROTOBUF_ALWAYS_INLINE void SyncHasbits( @@ -432,7 +451,7 @@ class PROTOBUF_EXPORT TcParser final { if (has_bits_offset) { // Only the first 32 has-bits are updated. Nothing above those is stored, // but e.g. messages without has-bits update the upper bits. - RefAt(msg, has_bits_offset) = static_cast(hasbits); + RefAt(msg, has_bits_offset) |= static_cast(hasbits); } } diff --git a/src/google/protobuf/generated_message_tctable_lite.cc b/src/google/protobuf/generated_message_tctable_lite.cc index 9993811dc..519b10776 100644 --- a/src/google/protobuf/generated_message_tctable_lite.cc +++ b/src/google/protobuf/generated_message_tctable_lite.cc @@ -85,10 +85,7 @@ PROTOBUF_NOINLINE const char* TcParser::ParseLoop( const TcParseTableBase* table) { ScopedArenaSwap saved(msg, ctx); while (!ctx->Done(&ptr)) { - // Unconditionally read has bits, even if we don't have has bits. - // has_bits_offset will be 0 and we will just read something valid. - uint64_t hasbits = ReadAt(msg, table->has_bits_offset); - ptr = TagDispatch(msg, ptr, ctx, table, hasbits, {}); + ptr = TagDispatch(msg, ptr, ctx, table, 0, {}); if (ptr == nullptr) break; if (ctx->LastTag() != 1) break; // Ended on terminating tag } @@ -362,9 +359,9 @@ inline PROTOBUF_ALWAYS_INLINE void InvertPacked(TcFieldData& data) { // Message fields ////////////////////////////////////////////////////////////////////////////// -template -inline PROTOBUF_ALWAYS_INLINE -const char* TcParser::SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { +template +inline PROTOBUF_ALWAYS_INLINE const char* TcParser::SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } @@ -373,74 +370,139 @@ const char* TcParser::SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { hasbits |= (uint64_t{1} << data.hasbit_idx()); SyncHasbits(msg, hasbits, table); auto& field = RefAt(msg, data.offset()); - if (field == nullptr) { - const MessageLite* default_instance = - table->field_aux(data.aux_idx())->message_default; - field = default_instance->New(ctx->data().arena); + + if (aux_is_table) { + const auto* inner_table = table->field_aux(data.aux_idx())->table; + if (field == nullptr) { + field = inner_table->default_instance->New(ctx->data().arena); + } + if (group_coding) { + return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag), + inner_table); + } + return ctx->ParseMessage(field, ptr, inner_table); + } else { + if (field == nullptr) { + const MessageLite* default_instance = + table->field_aux(data.aux_idx())->message_default(); + field = default_instance->New(ctx->data().arena); + } + if (group_coding) { + return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag)); + } + return ctx->ParseMessage(field, ptr); } - if (group_coding) { - return ctx->ParseGroup(field, ptr, FastDecodeTag(saved_tag)); - } - return ctx->ParseMessage(field, ptr); } -const char* TcParser::FastMS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( +const char* TcParser::FastMdS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastMS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( +const char* TcParser::FastMdS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastGS1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( +const char* TcParser::FastGdS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastGS2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( +const char* TcParser::FastGdS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -template -inline PROTOBUF_ALWAYS_INLINE -const char* TcParser::RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL) { +const char* TcParser::FastMtS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastMtS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGtS1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGtS2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return SingularParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +template +inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_DECL) { if (PROTOBUF_PREDICT_FALSE(data.coded_tag() != 0)) { PROTOBUF_MUSTTAIL return MiniParse(PROTOBUF_TC_PARAM_PASS); } auto saved_tag = UnalignedLoad(ptr); ptr += sizeof(TagType); SyncHasbits(msg, hasbits, table); - const MessageLite* default_instance = - table->field_aux(data.aux_idx())->message_default; - auto& field = RefAt(msg, data.offset()); - MessageLite* submsg = - field.Add>(default_instance); - if (group_coding) { - return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag)); + auto* aux = table->field_aux(data.aux_idx()); + if (aux_is_table) { + auto* inner_table = aux->table; + auto& field = RefAt(msg, data.offset()); + MessageLite* submsg = field.Add>( + inner_table->default_instance); + if (group_coding) { + return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag), + inner_table); + } + return ctx->ParseMessage(submsg, ptr, inner_table); + } else { + const MessageLite* default_instance = aux->message_default(); + auto& field = RefAt(msg, data.offset()); + MessageLite* submsg = + field.Add>(default_instance); + if (group_coding) { + return ctx->ParseGroup(submsg, ptr, FastDecodeTag(saved_tag)); + } + return ctx->ParseMessage(submsg, ptr); } - return ctx->ParseMessage(submsg, ptr); } -const char* TcParser::FastMR1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( +const char* TcParser::FastMdR1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastMR2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( +const char* TcParser::FastMdR2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastGR1(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( +const char* TcParser::FastGdR1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } -const char* TcParser::FastGR2(PROTOBUF_TC_PARAM_DECL) { - PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( +const char* TcParser::FastGdR2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastMtR1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastMtR2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGtR1(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( + PROTOBUF_TC_PARAM_PASS); +} + +const char* TcParser::FastGtR2(PROTOBUF_TC_PARAM_DECL) { + PROTOBUF_MUSTTAIL return RepeatedParseMessageAuxImpl( PROTOBUF_TC_PARAM_PASS); } @@ -977,10 +1039,6 @@ const char* TcParser::FastZ64P2(PROTOBUF_TC_PARAM_DECL) { PROTOBUF_NOINLINE const char* TcParser::FastUnknownEnumFallback( PROTOBUF_TC_PARAM_DECL) { - (void)msg; - (void)ctx; - (void)hasbits; - // If we know we want to put this field directly into the unknown field set, // then we can skip the call to MiniParse and directly call table->fallback. // However, we first have to update `data` to contain the decoded tag. @@ -1261,20 +1319,14 @@ const char* TcParser::FastUR2(PROTOBUF_TC_PARAM_DECL) { ////////////////////////////////////////////////////////////////////////////// namespace { -inline void SetHas(const TcParseTableBase* table, const FieldEntry& entry, - MessageLite* msg, uint64_t& hasbits) { - int32_t has_idx = entry.has_idx; - if (has_idx < 32) { - hasbits |= uint64_t{1} << has_idx; - } else { - auto* hasblocks = &TcParser::RefAt(msg, table->has_bits_offset); +inline void SetHas(const FieldEntry& entry, MessageLite* msg) { + auto has_idx = static_cast(entry.has_idx); #if defined(__x86_64__) && defined(__GNUC__) - asm("bts %1, %0\n" : "+m"(*hasblocks) : "r"(has_idx)); + asm("bts %1, %0\n" : "+m"(*msg) : "r"(has_idx)); #else - auto& hasblock = hasblocks[has_idx / 32]; - hasblock |= uint32_t{1} << (has_idx % 32); + auto& hasblock = TcParser::RefAt(msg, has_idx / 32 * 4); + hasblock |= uint32_t{1} << (has_idx % 32); #endif - } } } // namespace @@ -1364,7 +1416,7 @@ const char* TcParser::MpFixed(PROTOBUF_TC_PARAM_DECL) { } // Set the field present: if (card == field_layout::kFcOptional) { - SetHas(table, entry, msg, hasbits); + SetHas(entry, msg); } else if (card == field_layout::kFcOneof) { ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); } @@ -1501,7 +1553,7 @@ const char* TcParser::MpVarint(PROTOBUF_TC_PARAM_DECL) { // Mark the field as present: const bool is_oneof = card == field_layout::kFcOneof; if (card == field_layout::kFcOptional) { - SetHas(table, entry, msg, hasbits); + SetHas(entry, msg); } else if (is_oneof) { ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); } @@ -1676,7 +1728,7 @@ const char* TcParser::MpString(PROTOBUF_TC_PARAM_DECL) { const bool is_oneof = card == field_layout::kFcOneof; bool need_init = false; if (card == field_layout::kFcOptional) { - SetHas(table, entry, msg, hasbits); + SetHas(entry, msg); } else if (is_oneof) { need_init = ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); } @@ -1789,21 +1841,31 @@ const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) { const bool is_oneof = card == field_layout::kFcOneof; bool need_init = false; if (card == field_layout::kFcOptional) { - SetHas(table, entry, msg, hasbits); + SetHas(entry, msg); } else if (is_oneof) { need_init = ChangeOneof(table, entry, data.tag() >> 3, ctx, msg); } - MessageLite*& field = RefAt(msg, entry.offset); - if (need_init || field == nullptr) { - const MessageLite* default_instance = - table->field_aux(&entry)->message_default; - field = default_instance->New(ctx->data().arena); - } SyncHasbits(msg, hasbits, table); - if (is_group) { - return ctx->ParseGroup(field, ptr, decoded_tag); + MessageLite*& field = RefAt(msg, entry.offset); + if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) { + auto* inner_table = table->field_aux(&entry)->table; + if (need_init || field == nullptr) { + field = inner_table->default_instance->New(ctx->data().arena); + } + if (is_group) { + return ctx->ParseGroup(field, ptr, decoded_tag, inner_table); + } + return ctx->ParseMessage(field, ptr, inner_table); + } else { + if (need_init || field == nullptr) { + field = + table->field_aux(&entry)->message_default()->New(ctx->data().arena); + } + if (is_group) { + return ctx->ParseGroup(field, ptr, decoded_tag); + } + return ctx->ParseMessage(field, ptr); } - return ctx->ParseMessage(field, ptr); } const char* TcParser::MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL) { @@ -1837,15 +1899,24 @@ const char* TcParser::MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL) { } SyncHasbits(msg, hasbits, table); - const MessageLite* default_instance = - table->field_aux(&entry)->message_default; - auto& field = RefAt(msg, entry.offset); - MessageLite* value = - field.Add>(default_instance); - if (is_group) { - return ctx->ParseGroup(value, ptr, decoded_tag); + if ((type_card & field_layout::kTvMask) == field_layout::kTvTable) { + auto* inner_table = table->field_aux(&entry)->table; + auto& field = RefAt(msg, entry.offset); + MessageLite* value = field.Add>( + inner_table->default_instance); + if (is_group) { + return ctx->ParseGroup(value, ptr, decoded_tag, inner_table); + } + return ctx->ParseMessage(value, ptr, inner_table); + } else { + auto& field = RefAt(msg, entry.offset); + MessageLite* value = field.Add>( + table->field_aux(&entry)->message_default()); + if (is_group) { + return ctx->ParseGroup(value, ptr, decoded_tag); + } + return ctx->ParseMessage(value, ptr); } - return ctx->ParseMessage(value, ptr); } const char* TcParser::MpMap(PROTOBUF_TC_PARAM_DECL) { diff --git a/src/google/protobuf/io/BUILD.bazel b/src/google/protobuf/io/BUILD.bazel new file mode 100644 index 000000000..7582f78bc --- /dev/null +++ b/src/google/protobuf/io/BUILD.bazel @@ -0,0 +1,146 @@ +# Protobuf IO library. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") + +package( + default_visibility = ["//visibility:public"], +) + +cc_library( + name = "io", + srcs = [ + "coded_stream.cc", + "zero_copy_stream.cc", + "zero_copy_stream_impl.cc", + "zero_copy_stream_impl_lite.cc", + ], + hdrs = [ + "coded_stream.h", + "zero_copy_stream.h", + "zero_copy_stream_impl.h", + "zero_copy_stream_impl_lite.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/io", + deps = [ + ":io_win32", + "//src/google/protobuf:arena", + "//src/google/protobuf/stubs:lite", + ], +) + +cc_library( + name = "printer", + srcs = ["printer.cc"], + hdrs = ["printer.h"], + copts = COPTS + select({ + "//build_defs:config_msvc": [], + "//conditions:default": ["-Wno-maybe-uninitialized"], + }), + include_prefix = "google/protobuf/io", + deps = [ + ":io", + "//src/google/protobuf/stubs", + ], +) + +cc_library( + name = "tokenizer", + srcs = [ + "strtod.cc", + "tokenizer.cc", + ], + hdrs = [ + "strtod.h", + "tokenizer.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/io", + deps = [ + ":io", + "//src/google/protobuf/stubs", + ], +) + +cc_library( + name = "gzip_stream", + srcs = ["gzip_stream.cc"], + hdrs = ["gzip_stream.h"], + copts = COPTS + select({ + "//build_defs:config_msvc": [], + "//conditions:default": ["-Wno-maybe-uninitialized"], + }), + include_prefix = "google/protobuf/io", + deps = [ + ":io", + "//src/google/protobuf/stubs", + ] + select({ + "//build_defs:config_msvc": [], + "//conditions:default": ["@zlib//:zlib"], + }), +) + +cc_library( + name = "io_win32", + srcs = ["io_win32.cc"], + hdrs = ["io_win32.h"], + copts = COPTS, + include_prefix = "google/protobuf/io", + visibility = ["//pkg:__pkg__"], + deps = [ + "//src/google/protobuf:arena", + "//src/google/protobuf/stubs:lite", + ], +) + +cc_test( + name = "io_test", + srcs = [ + "coded_stream_unittest.cc", + "printer_unittest.cc", + "tokenizer_unittest.cc", + "zero_copy_stream_unittest.cc", + ], + copts = COPTS + select({ + "//build_defs:config_msvc": [], + "//conditions:default": ["-Wno-maybe-uninitialized"], + }), + data = [ + "//src/google/protobuf:testdata", + ], + deps = [ + ":gzip_stream", + ":io", + "//:protobuf", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "win32_test", + srcs = ["io_win32_unittest.cc"], + tags = [ + "manual", + "windows", + ], + deps = [ + "//:protobuf_lite", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index c8fc994f9..a8b5eb57b 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -109,7 +109,6 @@ #ifndef GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ #define GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ - #include #include diff --git a/src/google/protobuf/io/gzip_stream.h b/src/google/protobuf/io/gzip_stream.h index 4cf71b6c3..539f77bd3 100644 --- a/src/google/protobuf/io/gzip_stream.h +++ b/src/google/protobuf/io/gzip_stream.h @@ -43,7 +43,6 @@ #ifndef GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__ #define GOOGLE_PROTOBUF_IO_GZIP_STREAM_H__ - #include #include #include diff --git a/src/google/protobuf/io/printer.h b/src/google/protobuf/io/printer.h index 92a4321c0..e30bfa690 100644 --- a/src/google/protobuf/io/printer.h +++ b/src/google/protobuf/io/printer.h @@ -37,7 +37,6 @@ #ifndef GOOGLE_PROTOBUF_IO_PRINTER_H__ #define GOOGLE_PROTOBUF_IO_PRINTER_H__ - #include #include #include diff --git a/src/google/protobuf/io/tokenizer.h b/src/google/protobuf/io/tokenizer.h index 4abab7e30..ca2747b4b 100644 --- a/src/google/protobuf/io/tokenizer.h +++ b/src/google/protobuf/io/tokenizer.h @@ -37,7 +37,6 @@ #ifndef GOOGLE_PROTOBUF_IO_TOKENIZER_H__ #define GOOGLE_PROTOBUF_IO_TOKENIZER_H__ - #include #include diff --git a/src/google/protobuf/io/zero_copy_stream.h b/src/google/protobuf/io/zero_copy_stream.h index 2041cbf64..f39267d46 100644 --- a/src/google/protobuf/io/zero_copy_stream.h +++ b/src/google/protobuf/io/zero_copy_stream.h @@ -107,7 +107,6 @@ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_H__ - #include diff --git a/src/google/protobuf/io/zero_copy_stream_impl.h b/src/google/protobuf/io/zero_copy_stream_impl.h index a385992f2..04978d18b 100644 --- a/src/google/protobuf/io/zero_copy_stream_impl.h +++ b/src/google/protobuf/io/zero_copy_stream_impl.h @@ -40,7 +40,6 @@ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_H__ - #include #include diff --git a/src/google/protobuf/io/zero_copy_stream_impl_lite.cc b/src/google/protobuf/io/zero_copy_stream_impl_lite.cc index b3dfd84c7..bea35c123 100644 --- a/src/google/protobuf/io/zero_copy_stream_impl_lite.cc +++ b/src/google/protobuf/io/zero_copy_stream_impl_lite.cc @@ -36,12 +36,16 @@ #include #include +#include #include #include #include #include +// Must be included last +#include + namespace google { namespace protobuf { namespace io { diff --git a/src/google/protobuf/io/zero_copy_stream_impl_lite.h b/src/google/protobuf/io/zero_copy_stream_impl_lite.h index cbda32871..557993907 100644 --- a/src/google/protobuf/io/zero_copy_stream_impl_lite.h +++ b/src/google/protobuf/io/zero_copy_stream_impl_lite.h @@ -44,10 +44,10 @@ #ifndef GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__ #define GOOGLE_PROTOBUF_IO_ZERO_COPY_STREAM_IMPL_LITE_H__ - #include #include #include +#include #include #include @@ -78,6 +78,10 @@ class PROTOBUF_EXPORT ArrayInputStream PROTOBUF_FUTURE_FINAL ArrayInputStream(const void* data, int size, int block_size = -1); ~ArrayInputStream() override = default; + // `ArrayInputStream` is neither copiable nor assignable + ArrayInputStream(const ArrayInputStream&) = delete; + ArrayInputStream& operator=(const ArrayInputStream&) = delete; + // implements ZeroCopyInputStream ---------------------------------- bool Next(const void** data, int* size) override; void BackUp(int count) override; @@ -93,8 +97,6 @@ class PROTOBUF_EXPORT ArrayInputStream PROTOBUF_FUTURE_FINAL int position_; int last_returned_size_; // How many bytes we returned last time Next() // was called (used for error checking only). - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayInputStream); }; // =================================================================== @@ -113,6 +115,10 @@ class PROTOBUF_EXPORT ArrayOutputStream PROTOBUF_FUTURE_FINAL ArrayOutputStream(void* data, int size, int block_size = -1); ~ArrayOutputStream() override = default; + // `ArrayOutputStream` is neither copiable nor assignable + ArrayOutputStream(const ArrayOutputStream&) = delete; + ArrayOutputStream& operator=(const ArrayOutputStream&) = delete; + // implements ZeroCopyOutputStream --------------------------------- bool Next(void** data, int* size) override; void BackUp(int count) override; @@ -126,8 +132,6 @@ class PROTOBUF_EXPORT ArrayOutputStream PROTOBUF_FUTURE_FINAL int position_; int last_returned_size_; // How many bytes we returned last time Next() // was called (used for error checking only). - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArrayOutputStream); }; // =================================================================== @@ -148,6 +152,10 @@ class PROTOBUF_EXPORT StringOutputStream PROTOBUF_FUTURE_FINAL explicit StringOutputStream(std::string* target); ~StringOutputStream() override = default; + // `StringOutputStream` is neither copiable nor assignable + StringOutputStream(const StringOutputStream&) = delete; + StringOutputStream& operator=(const StringOutputStream&) = delete; + // implements ZeroCopyOutputStream --------------------------------- bool Next(void** data, int* size) override; void BackUp(int count) override; @@ -157,8 +165,6 @@ class PROTOBUF_EXPORT StringOutputStream PROTOBUF_FUTURE_FINAL static constexpr size_t kMinimumSize = 16; std::string* target_; - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(StringOutputStream); }; // Note: There is no StringInputStream. Instead, just create an @@ -215,6 +221,10 @@ class PROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream { int block_size = -1); ~CopyingInputStreamAdaptor() override; + // `CopyingInputStreamAdaptor` is neither copiable nor assignable + CopyingInputStreamAdaptor(const CopyingInputStreamAdaptor&) = delete; + CopyingInputStreamAdaptor& operator=(const CopyingInputStreamAdaptor&) = delete; + // Call SetOwnsCopyingStream(true) to tell the CopyingInputStreamAdaptor to // delete the underlying CopyingInputStream when it is destroyed. void SetOwnsCopyingStream(bool value) { owns_copying_stream_ = value; } @@ -255,8 +265,6 @@ class PROTOBUF_EXPORT CopyingInputStreamAdaptor : public ZeroCopyInputStream { // BackUp(). These need to be returned again. // 0 <= backup_bytes_ <= buffer_used_ int backup_bytes_; - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingInputStreamAdaptor); }; // =================================================================== @@ -298,6 +306,10 @@ class PROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStream { int block_size = -1); ~CopyingOutputStreamAdaptor() override; + // `CopyingOutputStreamAdaptor` is neither copiable nor assignable + CopyingOutputStreamAdaptor(const CopyingOutputStreamAdaptor&) = delete; + CopyingOutputStreamAdaptor& operator=(const CopyingOutputStreamAdaptor&) = delete; + // Writes all pending data to the underlying stream. Returns false if a // write error occurred on the underlying stream. (The underlying // stream itself is not necessarily flushed.) @@ -342,8 +354,6 @@ class PROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStream { // returned by Next()). When BackUp() is called, we just reduce this. // 0 <= buffer_used_ <= buffer_size_. int buffer_used_; - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CopyingOutputStreamAdaptor); }; // =================================================================== @@ -356,6 +366,10 @@ class PROTOBUF_EXPORT LimitingInputStream PROTOBUF_FUTURE_FINAL LimitingInputStream(ZeroCopyInputStream* input, int64_t limit); ~LimitingInputStream() override; + // `LimitingInputStream` is neither copiable nor assignable + LimitingInputStream(const LimitingInputStream&) = delete; + LimitingInputStream& operator=(const LimitingInputStream&) = delete; + // implements ZeroCopyInputStream ---------------------------------- bool Next(const void** data, int* size) override; void BackUp(int count) override; @@ -367,8 +381,6 @@ class PROTOBUF_EXPORT LimitingInputStream PROTOBUF_FUTURE_FINAL ZeroCopyInputStream* input_; int64_t limit_; // Decreases as we go, becomes negative if we overshoot. int64_t prior_bytes_read_; // Bytes read on underlying stream at construction - - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(LimitingInputStream); }; diff --git a/src/google/protobuf/io/zero_copy_stream_unittest.cc b/src/google/protobuf/io/zero_copy_stream_unittest.cc index d4e5b5451..5565e3869 100644 --- a/src/google/protobuf/io/zero_copy_stream_unittest.cc +++ b/src/google/protobuf/io/zero_copy_stream_unittest.cc @@ -61,6 +61,7 @@ #include #include +#include #include #include @@ -570,7 +571,7 @@ TEST_F(IoTest, CompressionOptions) { // Some ad-hoc testing of compression options. std::string golden_filename = - TestUtil::GetTestDataPath("net/proto2/internal/testdata/golden_message"); + TestUtil::GetTestDataPath("third_party/protobuf/testdata/golden_message"); std::string golden; GOOGLE_CHECK_OK(File::GetContents(golden_filename, &golden, true)); diff --git a/src/google/protobuf/map_test.inc b/src/google/protobuf/map_test.inc index 18a7bfb75..550d986af 100644 --- a/src/google/protobuf/map_test.inc +++ b/src/google/protobuf/map_test.inc @@ -3749,7 +3749,7 @@ TEST(MapSerializationTest, DeterministicSubmessage) { const std::string filename = "golden_message_maps"; std::string golden; GOOGLE_CHECK_OK(File::GetContents( - TestUtil::GetTestDataPath("net/proto2/internal/testdata/" + filename), + TestUtil::GetTestDataPath("third_party/protobuf/testdata/" + filename), &golden, true)); t.ParseFromString(golden); *(p.mutable_m()) = t; @@ -3791,7 +3791,7 @@ TEST(TextFormatMapTest, DynamicMessage) { std::string expected_text; GOOGLE_CHECK_OK( - File::GetContents(TestUtil::GetTestDataPath("net/proto2/internal/" + File::GetContents(TestUtil::GetTestDataPath("third_party/protobuf/" "testdata/map_test_data.txt"), &expected_text, true)); @@ -3808,7 +3808,7 @@ TEST(TextFormatMapTest, Sorted) { std::string expected_text; GOOGLE_CHECK_OK( - File::GetContents(TestUtil::GetTestDataPath("net/proto2/internal/" + File::GetContents(TestUtil::GetTestDataPath("third_party/protobuf/" "testdata/map_test_data.txt"), &expected_text, true)); @@ -3828,10 +3828,10 @@ TEST(TextFormatMapTest, Sorted) { TEST(TextFormatMapTest, ParseCorruptedString) { std::string serialized_message; - GOOGLE_CHECK_OK( - File::GetContents(TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_message_maps"), - &serialized_message, true)); + GOOGLE_CHECK_OK(File::GetContents( + TestUtil::GetTestDataPath( + "third_party/protobuf/testdata/golden_message_maps"), + &serialized_message, true)); UNITTEST::TestMaps message; GOOGLE_CHECK(message.ParseFromString(serialized_message)); TestParseCorruptedString(message); diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 39ec154c3..d20963bc6 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -1197,6 +1197,8 @@ class PROTOBUF_EXPORT Reflection final { void SwapOneofField(Message* lhs, Message* rhs, const OneofDescriptor* oneof_descriptor) const; + void InternalSwap(Message* lhs, Message* rhs) const; + inline bool HasOneofField(const Message& message, const FieldDescriptor* field) const; inline void SetOneofCase(Message* message, diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index 3a1b67bf6..27a8b38ba 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include #include diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index fa2bbbf59..19cc9c148 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -124,7 +124,7 @@ TEST(MESSAGE_TEST_NAME, SerializeToBrokenOstream) { TEST(MESSAGE_TEST_NAME, ParseFromFileDescriptor) { std::string filename = - TestUtil::GetTestDataPath("net/proto2/internal/testdata/golden_message"); + TestUtil::GetTestDataPath("third_party/protobuf/testdata/golden_message"); int file = open(filename.c_str(), O_RDONLY | O_BINARY); ASSERT_GE(file, 0); @@ -137,7 +137,7 @@ TEST(MESSAGE_TEST_NAME, ParseFromFileDescriptor) { TEST(MESSAGE_TEST_NAME, ParsePackedFromFileDescriptor) { std::string filename = TestUtil::GetTestDataPath( - "net/proto2/internal/testdata/golden_packed_fields_message"); + "third_party/protobuf/testdata/golden_packed_fields_message"); int file = open(filename.c_str(), O_RDONLY | O_BINARY); ASSERT_GE(file, 0); diff --git a/src/google/protobuf/parse_context.cc b/src/google/protobuf/parse_context.cc index 59852fdb7..4211babef 100644 --- a/src/google/protobuf/parse_context.cc +++ b/src/google/protobuf/parse_context.cc @@ -279,7 +279,8 @@ const char* ParseContext::ReadSizeAndPushLimitAndDepth(const char* ptr, const char* ParseContext::ParseMessage(MessageLite* msg, const char* ptr) { int old; ptr = ReadSizeAndPushLimitAndDepth(ptr, &old); - ptr = ptr ? msg->_InternalParse(ptr, this) : nullptr; + if (ptr == nullptr) return ptr; + ptr = msg->_InternalParse(ptr, this); depth_++; if (!PopLimit(old)) return nullptr; return ptr; diff --git a/src/google/protobuf/parse_context.h b/src/google/protobuf/parse_context.h index 7aea50cdc..96ee32095 100644 --- a/src/google/protobuf/parse_context.h +++ b/src/google/protobuf/parse_context.h @@ -427,6 +427,17 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { bool>::type = true> PROTOBUF_NODISCARD const char* ParseMessage(T* msg, const char* ptr); + template + PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* ParseMessage( + MessageLite* msg, const char* ptr, const Table* table) { + int old; + ptr = ReadSizeAndPushLimitAndDepthInlined(ptr, &old); + ptr = ptr ? TcParser::ParseLoop(msg, ptr, this, table) : nullptr; + depth_++; + if (!PopLimit(old)) return nullptr; + return ptr; + } + template PROTOBUF_NODISCARD PROTOBUF_NDEBUG_INLINE const char* ParseGroup( T* msg, const char* ptr, uint32_t tag) { @@ -439,6 +450,18 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { return ptr; } + template + PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* ParseGroup( + MessageLite* msg, const char* ptr, uint32_t tag, const Table* table) { + if (--depth_ < 0) return nullptr; + group_depth_++; + ptr = TcParser::ParseLoop(msg, ptr, this, table); + group_depth_--; + depth_++; + if (PROTOBUF_PREDICT_FALSE(!ConsumeEndGroup(tag))) return nullptr; + return ptr; + } + private: // Out-of-line routine to save space in ParseContext::ParseMessage // int old; @@ -451,6 +474,11 @@ class PROTOBUF_EXPORT ParseContext : public EpsCopyInputStream { PROTOBUF_NODISCARD const char* ReadSizeAndPushLimitAndDepth(const char* ptr, int* old_limit); + // As above, but fully inlined for the cases where we care about performance + // more than size. eg TcParser. + PROTOBUF_NODISCARD PROTOBUF_ALWAYS_INLINE const char* + ReadSizeAndPushLimitAndDepthInlined(const char* ptr, int* old_limit); + // The context keeps an internal stack to keep track of the recursive // part of the parse state. // Current depth of the active parser, depth counts down. @@ -749,12 +777,26 @@ PROTOBUF_NODISCARD const char* ParseContext::ParseMessage(T* msg, const char* ptr) { int old; ptr = ReadSizeAndPushLimitAndDepth(ptr, &old); - ptr = ptr ? msg->_InternalParse(ptr, this) : nullptr; + if (ptr == nullptr) return ptr; + ptr = msg->_InternalParse(ptr, this); depth_++; if (!PopLimit(old)) return nullptr; return ptr; } +inline const char* ParseContext::ReadSizeAndPushLimitAndDepthInlined( + const char* ptr, int* old_limit) { + int size = ReadSize(&ptr); + if (PROTOBUF_PREDICT_FALSE(!ptr)) { + // Make sure this isn't uninitialized even on error return + *old_limit = 0; + return nullptr; + } + *old_limit = PushLimit(ptr, size); + if (--depth_ < 0) return nullptr; + return ptr; +} + template const char* EpsCopyInputStream::ReadRepeatedFixed(const char* ptr, Tag expected_tag, diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc index dcb0ff758..033563090 100644 --- a/src/google/protobuf/port_def.inc +++ b/src/google/protobuf/port_def.inc @@ -178,17 +178,17 @@ #ifdef PROTOBUF_VERSION #error PROTOBUF_VERSION was previously defined #endif -#define PROTOBUF_VERSION 3020001 +#define PROTOBUF_VERSION 3021000 #ifdef PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC #error PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC was previously defined #endif -#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 3020000 +#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC 3021000 #ifdef PROTOBUF_MIN_PROTOC_VERSION #error PROTOBUF_MIN_PROTOC_VERSION was previously defined #endif -#define PROTOBUF_MIN_PROTOC_VERSION 3020000 +#define PROTOBUF_MIN_PROTOC_VERSION 3021000 #ifdef PROTOBUF_VERSION_SUFFIX #error PROTOBUF_VERSION_SUFFIX was previously defined @@ -385,7 +385,7 @@ // The minimum library version which works with the current version of the // headers. -#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 3020000 +#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION 3021000 #ifdef PROTOBUF_RTTI #error PROTOBUF_RTTI was previously defined @@ -650,7 +650,7 @@ (!defined(__APPLE__) && __clang_major__ >= 12)) #define PROTOBUF_CONSTINIT [[clang::require_constant_initialization]] #define PROTOBUF_CONSTEXPR constexpr -#elif PROTOBUF_GNUC_MIN(12, 0) +#elif PROTOBUF_GNUC_MIN(12, 2) #define PROTOBUF_CONSTINIT __constinit #define PROTOBUF_CONSTEXPR constexpr #else diff --git a/src/google/protobuf/source_context.pb.h b/src/google/protobuf/source_context.pb.h index 49ecc2e79..3625ece1f 100644 --- a/src/google/protobuf/source_context.pb.h +++ b/src/google/protobuf/source_context.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/struct.pb.h b/src/google/protobuf/struct.pb.h index 3a88ef75e..a8019f86b 100644 --- a/src/google/protobuf/struct.pb.h +++ b/src/google/protobuf/struct.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/stubs/BUILD.bazel b/src/google/protobuf/stubs/BUILD.bazel new file mode 100644 index 000000000..2446352ec --- /dev/null +++ b/src/google/protobuf/stubs/BUILD.bazel @@ -0,0 +1,138 @@ +# Protobuf stubs library. +# These are utilities that mirror the behavior of internal Google code. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") + +package( + default_visibility = ["//:__subpackages__"], +) + +cc_library( + name = "lite", + srcs = [ + "bytestream.cc", + "common.cc", + "int128.cc", + "status.cc", + "statusor.cc", + "stringpiece.cc", + "stringprintf.cc", + "structurally_valid.cc", + "strutil.cc", + "time.cc", + ], + hdrs = [ + "bytestream.h", + "callback.h", + "casts.h", + "common.h", + "hash.h", + "int128.h", + "logging.h", + "macros.h", + "map_util.h", + "mathutil.h", + "mutex.h", + "once.h", + "platform_macros.h", + "port.h", + "status.h", + "status_macros.h", + "statusor.h", + "stl_util.h", + "stringpiece.h", + "stringprintf.h", + "strutil.h", + "template_util.h", + "time.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/stubs", + linkopts = LINK_OPTS, + deps = ["//src/google/protobuf:port_def"], +) + +cc_library( + name = "stubs", + srcs = [ + "substitute.cc", + ], + hdrs = [ + "substitute.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/stubs", + textual_hdrs = [ + "bytestream.h", + "callback.h", + "casts.h", + "common.h", + "hash.h", + "int128.h", + "logging.h", + "macros.h", + "map_util.h", + "mathutil.h", + "mutex.h", + "once.h", + "platform_macros.h", + "port.h", + "status.h", + "status_macros.h", + "statusor.h", + "stl_util.h", + "stringpiece.h", + "stringprintf.h", + "strutil.h", + "template_util.h", + "time.h", + ], + deps = [ + ":lite", + "//src/google/protobuf:port_def", + ], +) + +cc_test( + name = "stubs_test", + srcs = [ + "bytestream_unittest.cc", + "common_unittest.cc", + "int128_unittest.cc", + "status_test.cc", + "statusor_test.cc", + "stringpiece_unittest.cc", + "stringprintf_unittest.cc", + "structurally_valid_unittest.cc", + "strutil_unittest.cc", + "template_util_unittest.cc", + "time_test.cc", + ], + copts = COPTS + select({ + "//build_defs:config_msvc": [], + "//conditions:default": [ + "-Wno-deprecated-declarations", + ], + }), + linkopts = LINK_OPTS, + deps = [ + ":lite", + ":stubs", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/stubs/common.h b/src/google/protobuf/stubs/common.h index c54468c90..8b581ba63 100644 --- a/src/google/protobuf/stubs/common.h +++ b/src/google/protobuf/stubs/common.h @@ -82,7 +82,7 @@ namespace internal { // The current version, represented as a single integer to make comparison // easier: major * 10^6 + minor * 10^3 + micro -#define GOOGLE_PROTOBUF_VERSION 3020001 +#define GOOGLE_PROTOBUF_VERSION 3021000 // A suffix string for alpha, beta or rc releases. Empty for stable releases. #define GOOGLE_PROTOBUF_VERSION_SUFFIX "-rc1" @@ -90,15 +90,15 @@ namespace internal { // The minimum header version which works with the current version of // the library. This constant should only be used by protoc's C++ code // generator. -static const int kMinHeaderVersionForLibrary = 3020000; +static const int kMinHeaderVersionForLibrary = 3021000; // The minimum protoc version which works with the current version of the // headers. -#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 3020000 +#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION 3021000 // The minimum header version which works with the current version of // protoc. This constant should only be used in VerifyVersion(). -static const int kMinHeaderVersionForProtoc = 3020000; +static const int kMinHeaderVersionForProtoc = 3021000; // Verifies that the headers and libraries are compatible. Use the macro // below to call this. diff --git a/src/google/protobuf/stubs/port.h b/src/google/protobuf/stubs/port.h index b074cb163..954132bd5 100644 --- a/src/google/protobuf/stubs/port.h +++ b/src/google/protobuf/stubs/port.h @@ -147,57 +147,6 @@ static const int64 kint64min = -kint64max - 1; static const uint32 kuint32max = 0xFFFFFFFFu; static const uint64 kuint64max = uint64_t{0xFFFFFFFFFFFFFFFFu}; -#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) ||\ - defined(MEMORY_SANITIZER) - -#ifdef __cplusplus -extern "C" { -#endif // __cplusplus -uint16_t __sanitizer_unaligned_load16(const void *p); -uint32_t __sanitizer_unaligned_load32(const void *p); -uint64_t __sanitizer_unaligned_load64(const void *p); -void __sanitizer_unaligned_store16(void *p, uint16_t v); -void __sanitizer_unaligned_store32(void *p, uint32_t v); -void __sanitizer_unaligned_store64(void *p, uint64_t v); -#ifdef __cplusplus -} // extern "C" -#endif // __cplusplus - -inline uint16_t GOOGLE_UNALIGNED_LOAD16(const void *p) { - return __sanitizer_unaligned_load16(p); -} - -inline uint32_t GOOGLE_UNALIGNED_LOAD32(const void *p) { - return __sanitizer_unaligned_load32(p); -} - -inline uint64_t GOOGLE_UNALIGNED_LOAD64(const void *p) { - return __sanitizer_unaligned_load64(p); -} - -inline void GOOGLE_UNALIGNED_STORE16(void *p, uint16_t v) { - __sanitizer_unaligned_store16(p, v); -} - -inline void GOOGLE_UNALIGNED_STORE32(void *p, uint32_t v) { - __sanitizer_unaligned_store32(p, v); -} - -inline void GOOGLE_UNALIGNED_STORE64(void *p, uint64_t v) { - __sanitizer_unaligned_store64(p, v); -} - -#elif defined(GOOGLE_PROTOBUF_USE_UNALIGNED) && GOOGLE_PROTOBUF_USE_UNALIGNED - -#define GOOGLE_UNALIGNED_LOAD16(_p) (*reinterpret_cast(_p)) -#define GOOGLE_UNALIGNED_LOAD32(_p) (*reinterpret_cast(_p)) -#define GOOGLE_UNALIGNED_LOAD64(_p) (*reinterpret_cast(_p)) - -#define GOOGLE_UNALIGNED_STORE16(_p, _val) (*reinterpret_cast(_p) = (_val)) -#define GOOGLE_UNALIGNED_STORE32(_p, _val) (*reinterpret_cast(_p) = (_val)) -#define GOOGLE_UNALIGNED_STORE64(_p, _val) (*reinterpret_cast(_p) = (_val)) - -#else inline uint16_t GOOGLE_UNALIGNED_LOAD16(const void *p) { uint16_t t; memcpy(&t, p, sizeof t); @@ -227,7 +176,6 @@ inline void GOOGLE_UNALIGNED_STORE32(void *p, uint32_t v) { inline void GOOGLE_UNALIGNED_STORE64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } -#endif #if defined(GOOGLE_PROTOBUF_OS_NACL) \ || (defined(__ANDROID__) && defined(__clang__) \ diff --git a/src/google/protobuf/test_util2.h b/src/google/protobuf/test_util2.h index 540af636c..27de06f42 100644 --- a/src/google/protobuf/test_util2.h +++ b/src/google/protobuf/test_util2.h @@ -42,11 +42,17 @@ namespace google { namespace protobuf { namespace TestUtil { -// Translate net/proto2/* -> google/protobuf/* +// Translate net/proto2/* or third_party/protobuf/* to google/protobuf/*. inline std::string TranslatePathToOpensource(const std::string& google3_path) { - const std::string prefix = "net/proto2/"; - GOOGLE_CHECK(google3_path.find(prefix) == 0) << google3_path; - std::string path = google3_path.substr(prefix.size()); + std::string net_proto2 = "net/proto2/"; + std::string third_party_protobuf = "third_party/protobuf/"; + std::string path; + if (google3_path.find(net_proto2) == 0) { + path = google3_path.substr(net_proto2.size()); + } else { + GOOGLE_CHECK(google3_path.find(third_party_protobuf) == 0) << google3_path; + path = google3_path.substr(third_party_protobuf.size()); + } path = StringReplace(path, "internal/", "", false); path = StringReplace(path, "proto/", "", false); diff --git a/src/google/protobuf/testing/BUILD.bazel b/src/google/protobuf/testing/BUILD.bazel new file mode 100644 index 000000000..3b87fe9c4 --- /dev/null +++ b/src/google/protobuf/testing/BUILD.bazel @@ -0,0 +1,41 @@ +# Protobuf testing support. +# This package contains testonly utilities used in C++ unit tests. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") + +package(default_visibility = ["//:__subpackages__"]) + +cc_library( + name = "testing", + testonly = 1, + srcs = [ + "file.cc", + "googletest.cc", + ], + hdrs = [ + "file.h", + "googletest.h", + ], + copts = COPTS, + include_prefix = "google/protobuf/testing", + linkopts = LINK_OPTS, + deps = [ + "//:protobuf_lite", # for ShutdownProtobufLibrary + "//src/google/protobuf/io", + "//src/google/protobuf/stubs:lite", + "@com_google_googletest//:gtest", + ], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index 19110499d..2a2cb84f4 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -905,16 +905,18 @@ class TextFormat::Parser::ParserImpl { return true; } if (TryConsume("[")) { - while (true) { - if (!LookingAt("{") && !LookingAt("<")) { - DO(SkipFieldValue()); - } else { - DO(SkipFieldMessage()); + if (!TryConsume("]")) { + while (true) { + if (!LookingAt("{") && !LookingAt("<")) { + DO(SkipFieldValue()); + } else { + DO(SkipFieldMessage()); + } + if (TryConsume("]")) { + break; + } + DO(Consume(",")); } - if (TryConsume("]")) { - break; - } - DO(Consume(",")); } ++recursion_limit_; return true; diff --git a/src/google/protobuf/text_format_unittest.cc b/src/google/protobuf/text_format_unittest.cc index 126f2f380..7d0dfdb34 100644 --- a/src/google/protobuf/text_format_unittest.cc +++ b/src/google/protobuf/text_format_unittest.cc @@ -94,7 +94,7 @@ class TextFormatTest : public testing::Test { static void SetUpTestSuite() { GOOGLE_CHECK_OK(File::GetContents( TestUtil::GetTestDataPath( - "net/proto2/internal/" + "third_party/protobuf/" "testdata/text_format_unittest_data_oneof_implemented.txt"), &static_proto_text_format_, true)); CleanStringLineEndings(&static_proto_text_format_, false); @@ -116,7 +116,7 @@ class TextFormatExtensionsTest : public testing::Test { public: static void SetUpTestSuite() { GOOGLE_CHECK_OK(File::GetContents( - TestUtil::GetTestDataPath("net/proto2/internal/testdata/" + TestUtil::GetTestDataPath("third_party/protobuf/testdata/" "text_format_unittest_extensions_data.txt"), &static_proto_text_format_, true)); CleanStringLineEndings(&static_proto_text_format_, false); @@ -1035,6 +1035,19 @@ TEST_F(TextFormatTest, ParseShortRepeatedConcatenatedWithEmpty) { EXPECT_EQ(4, proto_.repeatedgroup(1).a()); } +TEST_F(TextFormatTest, ParseShortRepeatedUnknownEmpty) { + std::string parse_string = + "repeated_string: \"before\"\n" + "unknown_field: []\n" + "repeated_string: \"after\"\n"; + TextFormat::Parser parser; + parser.AllowUnknownField(true); + + ASSERT_TRUE(parser.ParseFromString(parse_string, &proto_)); + + EXPECT_EQ(2, proto_.repeated_string_size()); +} + TEST_F(TextFormatTest, Comments) { // Test that comments are ignored. diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index 9f9afde58..01c8c8500 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index ec06f2de5..ecdb9d9ed 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/src/google/protobuf/unittest_mset.proto b/src/google/protobuf/unittest_mset.proto index 32949947a..dd8a3afbf 100644 --- a/src/google/protobuf/unittest_mset.proto +++ b/src/google/protobuf/unittest_mset.proto @@ -53,6 +53,11 @@ message NestedTestMessageSetContainer { optional NestedTestMessageSetContainer child = 2; } +message NestedTestInt { + optional fixed32 a = 1; + optional NestedTestInt child = 2; +} + message TestMessageSetExtension1 { extend proto2_wireformat_unittest.TestMessageSet { optional TestMessageSetExtension1 message_set_extension = 1545008; @@ -69,11 +74,6 @@ message TestMessageSetExtension2 { optional string str = 25; } -message NestedTestInt { - optional fixed32 a = 1; - optional NestedTestInt child = 2; -} - message TestMessageSetExtension3 { extend proto2_wireformat_unittest.TestMessageSet { optional TestMessageSetExtension3 message_set_extension = 195273129; diff --git a/src/google/protobuf/util/BUILD.bazel b/src/google/protobuf/util/BUILD.bazel new file mode 100644 index 000000000..4fb8319d2 --- /dev/null +++ b/src/google/protobuf/util/BUILD.bazel @@ -0,0 +1,282 @@ +################################################################################ +# Protocol Buffers C++ Utilities +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_proto_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("//build_defs:cpp_opts.bzl", "COPTS") + +cc_library( + name = "delimited_message_util", + srcs = ["delimited_message_util.cc"], + hdrs = ["delimited_message_util.h"], + copts = COPTS, + strip_include_prefix = "/src", + visibility = ["//:__subpackages__"], + deps = [ + "//:protobuf_lite", + "//src/google/protobuf/io", + ], +) + +cc_test( + name = "delimited_message_util_test", + srcs = ["delimited_message_util_test.cc"], + copts = COPTS, + deps = [ + ":delimited_message_util", + "//src/google/protobuf:cc_test_protos", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "differencer", + srcs = [ + "field_comparator.cc", + "message_differencer.cc", + ], + hdrs = [ + "field_comparator.h", + "message_differencer.h", + ], + strip_include_prefix = "/src", + visibility = ["//:__subpackages__"], + deps = [ + "//src/google/protobuf", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + ], +) + +cc_test( + name = "field_comparator_test", + srcs = ["field_comparator_test.cc"], + deps = [ + ":differencer", + ":message_differencer_unittest_cc_proto", + "//src/google/protobuf:cc_test_protos", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "message_differencer_unittest", + srcs = ["message_differencer_unittest.cc"], + copts = COPTS + [ + "-Wno-deprecated-declarations", + ], + deps = [ + ":differencer", + ":message_differencer_unittest_cc_proto", + "//src/google/protobuf:cc_test_protos", + "//src/google/protobuf:test_util", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "field_mask_util", + srcs = ["field_mask_util.cc"], + hdrs = ["field_mask_util.h"], + copts = COPTS, + strip_include_prefix = "/src", + visibility = ["//:__subpackages__"], + deps = [ + "//src/google/protobuf", + "//src/google/protobuf/stubs", + ], +) + +cc_test( + name = "field_mask_util_test", + srcs = ["field_mask_util_test.cc"], + copts = COPTS, + deps = [ + ":field_mask_util", + "//src/google/protobuf", + "//src/google/protobuf:cc_test_protos", + "//src/google/protobuf:test_util", + "//src/google/protobuf/stubs", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "json_util", + srcs = ["json_util.cc"], + hdrs = ["json_util.h"], + copts = COPTS, + strip_include_prefix = "/src", + visibility = ["//:__subpackages__"], + deps = [ + ":type_resolver_util", + "//src/google/protobuf", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + "//src/google/protobuf/util/internal:default_value", + "//src/google/protobuf/util/internal:json", + "//src/google/protobuf/util/internal:protostream", + "//src/google/protobuf/util/internal:utility", + ], +) + +cc_test( + name = "json_util_test", + srcs = ["json_util_test.cc"], + copts = COPTS, + data = [ + "//src/google/protobuf:testdata", + ], + deps = [ + ":json_format_cc_proto", + ":json_format_proto3_cc_proto", + ":json_util", + "//src/google/protobuf", + "//src/google/protobuf:cc_test_protos", + "//src/google/protobuf:test_util", + "//src/google/protobuf/testing", + "//src/google/protobuf/util/internal:maps_cc_proto", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "time_util", + srcs = ["time_util.cc"], + hdrs = ["time_util.h"], + strip_include_prefix = "/src", + visibility = ["//:__subpackages__"], + deps = [ + "//src/google/protobuf", + "//src/google/protobuf/stubs", + ], +) + +cc_test( + name = "time_util_test", + srcs = ["time_util_test.cc"], + deps = [ + ":time_util", + "//src/google/protobuf", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "type_resolver_util", + srcs = ["type_resolver_util.cc"], + hdrs = [ + "type_resolver.h", + "type_resolver_util.h", + ], + copts = COPTS, + strip_include_prefix = "/src", + visibility = ["//:__subpackages__"], + deps = [ + "//src/google/protobuf", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + "//src/google/protobuf/util/internal:utility", + ], +) + +cc_test( + name = "type_resolver_util_test", + srcs = ["type_resolver_util_test.cc"], + copts = COPTS, + deps = [ + ":json_format_cc_proto", + ":json_format_proto3_cc_proto", + ":json_util", + "//src/google/protobuf", + "//src/google/protobuf:test_util", + "//src/google/protobuf/testing", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +# Testonly protos + +filegroup( + name = "test_proto_srcs", + testonly = 1, + srcs = [ + "json_format.proto", + "json_format_proto3.proto", + ], + visibility = ["//python:__pkg__"], +) + +proto_library( + name = "json_format_proto", + testonly = 1, + srcs = ["json_format.proto"], + strip_import_prefix = "/src", +) + +cc_proto_library( + name = "json_format_cc_proto", + testonly = 1, + deps = [":json_format_proto"], +) + +proto_library( + name = "json_format_proto3_proto", + testonly = 1, + srcs = ["json_format_proto3.proto"], + strip_import_prefix = "/src", + deps = [ + "//:any_proto", + "//:duration_proto", + "//:field_mask_proto", + "//:struct_proto", + "//:test_protos", + "//:timestamp_proto", + "//:wrappers_proto", + ], +) + +cc_proto_library( + name = "json_format_proto3_cc_proto", + testonly = 1, + deps = [":json_format_proto3_proto"], +) + +proto_library( + name = "message_differencer_unittest_proto", + testonly = 1, + srcs = ["message_differencer_unittest.proto"], + strip_import_prefix = "/src", + deps = ["//:any_proto"], +) + +cc_proto_library( + name = "message_differencer_unittest_cc_proto", + testonly = 1, + deps = [":message_differencer_unittest_proto"], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/util/internal/BUILD.bazel b/src/google/protobuf/util/internal/BUILD.bazel new file mode 100644 index 000000000..d42ecad51 --- /dev/null +++ b/src/google/protobuf/util/internal/BUILD.bazel @@ -0,0 +1,450 @@ +################################################################################ +# Protocol Buffers C++ util internals. +################################################################################ + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_proto_library", "cc_test") +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("//build_defs:cpp_opts.bzl", "COPTS") + +package( + default_visibility = ["//src/google/protobuf/util:__pkg__"], +) + +cc_library( + name = "constants", + hdrs = ["constants.h"], + strip_include_prefix = "/src", + deps = [ + "//src/google/protobuf/stubs", + ], +) + +cc_library( + name = "datapiece", + srcs = ["datapiece.cc"], + hdrs = ["datapiece.h"], + copts = COPTS, + strip_include_prefix = "/src", + deps = [ + ":constants", + ":utility", + "//src/google/protobuf", + "//src/google/protobuf/stubs", + ], +) + +cc_library( + name = "default_value", + srcs = ["default_value_objectwriter.cc"], + hdrs = ["default_value_objectwriter.h"], + copts = COPTS, + strip_include_prefix = "/src", + deps = [ + ":datapiece", + ":object_writer", + ":type_info", + ":utility", + "//src/google/protobuf/stubs", + ], +) + +cc_test( + name = "default_value_objectwriter_test", + srcs = ["default_value_objectwriter_test.cc"], + copts = COPTS, + deps = [ + ":default_value", + ":default_value_test_cc_proto", + ":expecting_objectwriter", + ":type_info_test_helper", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "expecting_objectwriter", + testonly = 1, + hdrs = ["expecting_objectwriter.h"], + strip_include_prefix = "/src", + deps = [":object_writer"], +) + +cc_library( + name = "field_mask_utility", + srcs = ["field_mask_utility.cc"], + hdrs = ["field_mask_utility.h"], + strip_include_prefix = "/src", + deps = [ + ":utility", + "//src/google/protobuf/stubs", + ], +) + +cc_library( + name = "json", + srcs = [ + "json_escaping.cc", + "json_objectwriter.cc", + "json_stream_parser.cc", + ], + hdrs = [ + "json_escaping.h", + "json_objectwriter.h", + "json_stream_parser.h", + ], + copts = COPTS, + strip_include_prefix = "/src", + deps = [ + ":object_writer", + ":utility", + "//src/google/protobuf", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + ], +) + +cc_test( + name = "json_objectwriter_test", + srcs = ["json_objectwriter_test.cc"], + copts = COPTS, + deps = [ + ":json", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "json_stream_parser_test", + srcs = ["json_stream_parser_test.cc"], + copts = COPTS, + deps = [ + ":expecting_objectwriter", + ":json", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "mock_error_listener", + testonly = 1, + hdrs = ["mock_error_listener.h"], + strip_include_prefix = "/src", + deps = [":protostream"], +) + +cc_library( + name = "object_writer", + srcs = ["object_writer.cc"], + hdrs = [ + "object_source.h", + "object_writer.h", + "structured_objectwriter.h", + ], + strip_include_prefix = "/src", + deps = [ + ":datapiece", + "//src/google/protobuf/stubs", + ], +) + +cc_library( + name = "type_info", + srcs = ["type_info.cc"], + hdrs = ["type_info.h"], + strip_include_prefix = "/src", + deps = [ + ":utility", + "//src/google/protobuf/stubs", + "//src/google/protobuf/util:type_resolver_util", + ], +) + +cc_library( + name = "type_info_test_helper", + testonly = 1, + srcs = ["type_info_test_helper.cc"], + hdrs = ["type_info_test_helper.h"], + copts = COPTS, + strip_include_prefix = "/src", + deps = [ + ":default_value", + ":protostream", + ":type_info", + ], +) + +cc_library( + name = "protostream", + srcs = [ + "error_listener.cc", + "proto_writer.cc", + "protostream_objectsource.cc", + "protostream_objectwriter.cc", + ], + hdrs = [ + "error_listener.h", + "location_tracker.h", + "object_location_tracker.h", + "proto_writer.h", + "protostream_objectsource.h", + "protostream_objectwriter.h", + ], + copts = COPTS, + strip_include_prefix = "/src", + deps = [ + ":constants", + ":datapiece", + ":field_mask_utility", + ":object_writer", + ":type_info", + ":utility", + "//src/google/protobuf", + "//src/google/protobuf/io", + "//src/google/protobuf/stubs", + "//src/google/protobuf/util:type_resolver_util", + ], +) + +cc_test( + name = "protostream_objectsource_test", + srcs = ["protostream_objectsource_test.cc"], + copts = COPTS, + deps = [ + ":anys_cc_proto", + ":books_cc_proto", + ":expecting_objectwriter", + ":field_mask_cc_proto", + ":maps_cc_proto", + ":oneofs_cc_proto", + ":proto3_cc_proto", + ":protostream", + ":struct_cc_proto", + ":timestamp_duration_cc_proto", + ":type_info_test_helper", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "protostream_objectwriter_test", + srcs = ["protostream_objectwriter_test.cc"], + copts = COPTS, + deps = [ + ":anys_cc_proto", + ":books_cc_proto", + ":field_mask_cc_proto", + ":maps_cc_proto", + ":mock_error_listener", + ":oneofs_cc_proto", + ":proto3_cc_proto", + ":protostream", + ":struct_cc_proto", + ":timestamp_duration_cc_proto", + ":type_info_test_helper", + ":wrappers_cc_proto", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "utility", + srcs = ["utility.cc"], + hdrs = ["utility.h"], + copts = COPTS, + strip_include_prefix = "/src", + deps = [ + ":constants", + "//src/google/protobuf", + "//src/google/protobuf/stubs", + ], +) + +# Testing protos: + +proto_library( + name = "anys_proto", + testonly = 1, + srcs = ["testdata/anys.proto"], + strip_import_prefix = "/src", + deps = [ + "//:any_proto", + "//:duration_proto", + "//:empty_proto", + "//:struct_proto", + "//:timestamp_proto", + "//:wrappers_proto", + ], +) + +cc_proto_library( + name = "anys_cc_proto", + testonly = 1, + visibility = ["//src/google/protobuf/util:__pkg__"], + deps = [":anys_proto"], +) + +proto_library( + name = "books_proto", + testonly = 1, + srcs = ["testdata/books.proto"], + strip_import_prefix = "/src", + deps = [":anys_proto"], +) + +cc_proto_library( + name = "books_cc_proto", + testonly = 1, + deps = [":books_proto"], +) + +proto_library( + name = "default_value_proto", + testonly = 1, + srcs = ["testdata/default_value.proto"], + strip_import_prefix = "/src", + deps = [ + "//:any_proto", + "//:struct_proto", + "//:wrappers_proto", + ], +) + +cc_proto_library( + name = "default_value_cc_proto", + testonly = 1, + deps = [":default_value_proto"], +) + +proto_library( + name = "default_value_test_proto", + testonly = 1, + srcs = ["testdata/default_value_test.proto"], + strip_import_prefix = "/src", +) + +cc_proto_library( + name = "default_value_test_cc_proto", + testonly = 1, + deps = [":default_value_test_proto"], +) + +proto_library( + name = "field_mask_proto", + testonly = 1, + srcs = ["testdata/field_mask.proto"], + strip_import_prefix = "/src", + deps = ["//:field_mask_proto"], +) + +cc_proto_library( + name = "field_mask_cc_proto", + testonly = 1, + deps = [":field_mask_proto"], +) + +proto_library( + name = "maps_proto", + testonly = 1, + srcs = ["testdata/maps.proto"], + strip_import_prefix = "/src", + deps = ["//:any_proto"], +) + +cc_proto_library( + name = "maps_cc_proto", + testonly = 1, + deps = [":maps_proto"], +) + +proto_library( + name = "oneofs_proto", + testonly = 1, + srcs = ["testdata/oneofs.proto"], + strip_import_prefix = "/src", + deps = [ + "//:any_proto", + "//:struct_proto", + "//:timestamp_proto", + ], +) + +cc_proto_library( + name = "oneofs_cc_proto", + testonly = 1, + deps = [":oneofs_proto"], +) + +proto_library( + name = "proto3_proto", + testonly = 1, + srcs = ["testdata/proto3.proto"], + strip_import_prefix = "/src", +) + +cc_proto_library( + name = "proto3_cc_proto", + testonly = 1, + deps = [":proto3_proto"], +) + +proto_library( + name = "struct_proto", + testonly = 1, + srcs = ["testdata/struct.proto"], + strip_import_prefix = "/src", + deps = ["//:struct_proto"], +) + +cc_proto_library( + name = "struct_cc_proto", + testonly = 1, + deps = [":struct_proto"], +) + +proto_library( + name = "timestamp_duration_proto", + testonly = 1, + srcs = ["testdata/timestamp_duration.proto"], + strip_import_prefix = "/src", + deps = [ + "//:duration_proto", + "//:timestamp_proto", + ], +) + +cc_proto_library( + name = "timestamp_duration_cc_proto", + testonly = 1, + deps = [":timestamp_duration_proto"], +) + +proto_library( + name = "wrappers_proto", + testonly = 1, + srcs = ["testdata/wrappers.proto"], + strip_import_prefix = "/src", + deps = ["//:wrappers_proto"], +) + +cc_proto_library( + name = "wrappers_cc_proto", + testonly = 1, + deps = [":wrappers_proto"], +) + +################################################################################ +# Distribution packaging +################################################################################ + +pkg_files( + name = "dist_files", + srcs = glob(["**/*"]), + strip_prefix = strip_prefix.from_root(""), + visibility = ["//src:__pkg__"], +) diff --git a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc index 96b14db0a..dc35e51ea 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc @@ -31,8 +31,8 @@ #include #include -#include #include +#include #include #include diff --git a/src/google/protobuf/util/internal/protostream_objectsource_test.cc b/src/google/protobuf/util/internal/protostream_objectsource_test.cc index e16db78a1..95e5f1633 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource_test.cc @@ -39,6 +39,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -46,10 +50,6 @@ #include #include #include -#include -#include -#include -#include #include diff --git a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc index 334899985..24040f4e0 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc @@ -40,6 +40,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include #include #include @@ -49,12 +55,6 @@ #include #include #include -#include -#include -#include -#include -#include -#include #include #include #include diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index 7216199e8..ac9408d8c 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -34,14 +34,14 @@ #include #include -#include -#include -#include #include #include #include #include #include +#include +#include +#include #include #include diff --git a/src/google/protobuf/util/message_differencer_unittest.cc b/src/google/protobuf/util/message_differencer_unittest.cc index f8e44df40..6e6dc6400 100644 --- a/src/google/protobuf/util/message_differencer_unittest.cc +++ b/src/google/protobuf/util/message_differencer_unittest.cc @@ -53,11 +53,11 @@ #include #include #include -#include #include #include #include #include +#include namespace google { namespace protobuf { diff --git a/src/google/protobuf/util/time_util.cc b/src/google/protobuf/util/time_util.cc index 9893aa35d..4c69b50be 100644 --- a/src/google/protobuf/util/time_util.cc +++ b/src/google/protobuf/util/time_util.cc @@ -50,19 +50,24 @@ using google::protobuf::Duration; using google::protobuf::Timestamp; namespace { -static const int kNanosPerSecond = 1000000000; -static const int kMicrosPerSecond = 1000000; -static const int kMillisPerSecond = 1000; -static const int kNanosPerMillisecond = 1000000; -static const int kNanosPerMicrosecond = 1000; -static const int kSecondsPerMinute = 60; // Note that we ignore leap seconds. -static const int kSecondsPerHour = 3600; +static constexpr int32_t kNanosPerSecond = 1000000000; +static constexpr int32_t kMicrosPerSecond = 1000000; +static constexpr int32_t kMillisPerSecond = 1000; +static constexpr int32_t kNanosPerMillisecond = 1000000; +static constexpr int32_t kNanosPerMicrosecond = 1000; +static constexpr int32_t kSecondsPerMinute = + 60; // Note that we ignore leap seconds. +static constexpr int32_t kSecondsPerHour = 3600; template -T CreateNormalized(int64_t seconds, int64_t nanos); +T CreateNormalized(int64_t seconds, int32_t nanos); template <> -Timestamp CreateNormalized(int64_t seconds, int64_t nanos) { +Timestamp CreateNormalized(int64_t seconds, int32_t nanos) { + GOOGLE_DCHECK(seconds >= TimeUtil::kTimestampMinSeconds && + seconds <= TimeUtil::kTimestampMaxSeconds) + << "Timestamp seconds are outside of the valid range"; + // Make sure nanos is in the range. if (nanos <= -kNanosPerSecond || nanos >= kNanosPerSecond) { seconds += nanos / kNanosPerSecond; @@ -73,8 +78,12 @@ Timestamp CreateNormalized(int64_t seconds, int64_t nanos) { seconds -= 1; nanos += kNanosPerSecond; } + GOOGLE_DCHECK(seconds >= TimeUtil::kTimestampMinSeconds && - seconds <= TimeUtil::kTimestampMaxSeconds); + seconds <= TimeUtil::kTimestampMaxSeconds && + nanos >= TimeUtil::kTimestampMinNanoseconds && + nanos <= TimeUtil::kTimestampMaxNanoseconds) + << "Timestamp is outside of the valid range"; Timestamp result; result.set_seconds(seconds); result.set_nanos(static_cast(nanos)); @@ -82,7 +91,11 @@ Timestamp CreateNormalized(int64_t seconds, int64_t nanos) { } template <> -Duration CreateNormalized(int64_t seconds, int64_t nanos) { +Duration CreateNormalized(int64_t seconds, int32_t nanos) { + GOOGLE_DCHECK(seconds >= TimeUtil::kDurationMinSeconds && + seconds <= TimeUtil::kDurationMaxSeconds) + << "Duration seconds are outside of the valid range"; + // Make sure nanos is in the range. if (nanos <= -kNanosPerSecond || nanos >= kNanosPerSecond) { seconds += nanos / kNanosPerSecond; @@ -96,8 +109,12 @@ Duration CreateNormalized(int64_t seconds, int64_t nanos) { seconds -= 1; nanos += kNanosPerSecond; } + GOOGLE_DCHECK(seconds >= TimeUtil::kDurationMinSeconds && - seconds <= TimeUtil::kDurationMaxSeconds); + seconds <= TimeUtil::kDurationMaxSeconds && + nanos >= TimeUtil::kDurationMinNanoseconds && + nanos <= TimeUtil::kDurationMaxNanoseconds) + << "Duration is outside of the valid range"; Duration result; result.set_seconds(seconds); result.set_nanos(static_cast(nanos)); @@ -150,8 +167,12 @@ int64_t RoundTowardZero(int64_t value, int64_t divider) { #ifndef _MSC_VER const int64_t TimeUtil::kTimestampMinSeconds; const int64_t TimeUtil::kTimestampMaxSeconds; +const int32_t TimeUtil::kTimestampMinNanoseconds; +const int32_t TimeUtil::kTimestampMaxNanoseconds; const int64_t TimeUtil::kDurationMaxSeconds; const int64_t TimeUtil::kDurationMinSeconds; +const int32_t TimeUtil::kDurationMaxNanoseconds; +const int32_t TimeUtil::kDurationMinNanoseconds; #endif // !_MSC_VER std::string TimeUtil::ToString(const Timestamp& timestamp) { @@ -261,37 +282,43 @@ Duration TimeUtil::SecondsToDuration(int64_t seconds) { } Duration TimeUtil::MinutesToDuration(int64_t minutes) { - return CreateNormalized(minutes * kSecondsPerMinute, 0); + GOOGLE_DCHECK(minutes >= TimeUtil::kDurationMinSeconds / kSecondsPerMinute && + minutes <= TimeUtil::kDurationMaxSeconds / kSecondsPerMinute) + << "Duration minutes are outside of the valid range"; + return SecondsToDuration(minutes * kSecondsPerMinute); } Duration TimeUtil::HoursToDuration(int64_t hours) { - return CreateNormalized(hours * kSecondsPerHour, 0); + GOOGLE_DCHECK(hours >= TimeUtil::kDurationMinSeconds / kSecondsPerHour && + hours <= TimeUtil::kDurationMaxSeconds / kSecondsPerHour) + << "Duration hours are outside of the valid range"; + return SecondsToDuration(hours * kSecondsPerHour); } int64_t TimeUtil::DurationToNanoseconds(const Duration& duration) { + GOOGLE_DCHECK(IsDurationValid(duration)) << "Duration is outside of the valid range"; return duration.seconds() * kNanosPerSecond + duration.nanos(); } int64_t TimeUtil::DurationToMicroseconds(const Duration& duration) { - return duration.seconds() * kMicrosPerSecond + - RoundTowardZero(duration.nanos(), kNanosPerMicrosecond); + return RoundTowardZero(DurationToNanoseconds(duration), kNanosPerMicrosecond); } int64_t TimeUtil::DurationToMilliseconds(const Duration& duration) { - return duration.seconds() * kMillisPerSecond + - RoundTowardZero(duration.nanos(), kNanosPerMillisecond); + return RoundTowardZero(DurationToNanoseconds(duration), kNanosPerMillisecond); } int64_t TimeUtil::DurationToSeconds(const Duration& duration) { + GOOGLE_DCHECK(IsDurationValid(duration)) << "Duration is outside of the valid range"; return duration.seconds(); } int64_t TimeUtil::DurationToMinutes(const Duration& duration) { - return RoundTowardZero(duration.seconds(), kSecondsPerMinute); + return RoundTowardZero(DurationToSeconds(duration), kSecondsPerMinute); } int64_t TimeUtil::DurationToHours(const Duration& duration) { - return RoundTowardZero(duration.seconds(), kSecondsPerHour); + return RoundTowardZero(DurationToSeconds(duration), kSecondsPerHour); } Timestamp TimeUtil::NanosecondsToTimestamp(int64_t nanos) { @@ -316,20 +343,28 @@ Timestamp TimeUtil::SecondsToTimestamp(int64_t seconds) { } int64_t TimeUtil::TimestampToNanoseconds(const Timestamp& timestamp) { + GOOGLE_DCHECK(IsTimestampValid(timestamp)) + << "Timestamp is outside of the valid range"; return timestamp.seconds() * kNanosPerSecond + timestamp.nanos(); } int64_t TimeUtil::TimestampToMicroseconds(const Timestamp& timestamp) { + GOOGLE_DCHECK(IsTimestampValid(timestamp)) + << "Timestamp is outside of the valid range"; return timestamp.seconds() * kMicrosPerSecond + RoundTowardZero(timestamp.nanos(), kNanosPerMicrosecond); } int64_t TimeUtil::TimestampToMilliseconds(const Timestamp& timestamp) { + GOOGLE_DCHECK(IsTimestampValid(timestamp)) + << "Timestamp is outside of the valid range"; return timestamp.seconds() * kMillisPerSecond + RoundTowardZero(timestamp.nanos(), kNanosPerMillisecond); } int64_t TimeUtil::TimestampToSeconds(const Timestamp& timestamp) { + GOOGLE_DCHECK(IsTimestampValid(timestamp)) + << "Timestamp is outside of the valid range"; return timestamp.seconds(); } diff --git a/src/google/protobuf/util/time_util.h b/src/google/protobuf/util/time_util.h index 709527ea2..bc45f70cd 100644 --- a/src/google/protobuf/util/time_util.h +++ b/src/google/protobuf/util/time_util.h @@ -69,11 +69,29 @@ class PROTOBUF_EXPORT TimeUtil { // The min/max Timestamp/Duration values we support. // // For "0001-01-01T00:00:00Z". - static const int64_t kTimestampMinSeconds = -62135596800LL; + static constexpr int64_t kTimestampMinSeconds = -62135596800LL; // For "9999-12-31T23:59:59.999999999Z". - static const int64_t kTimestampMaxSeconds = 253402300799LL; - static const int64_t kDurationMinSeconds = -315576000000LL; - static const int64_t kDurationMaxSeconds = 315576000000LL; + static constexpr int64_t kTimestampMaxSeconds = 253402300799LL; + static constexpr int32_t kTimestampMinNanoseconds = 0; + static constexpr int32_t kTimestampMaxNanoseconds = 999999999; + static constexpr int64_t kDurationMinSeconds = -315576000000LL; + static constexpr int64_t kDurationMaxSeconds = 315576000000LL; + static constexpr int32_t kDurationMinNanoseconds = -999999999; + static constexpr int32_t kDurationMaxNanoseconds = 999999999; + + static bool IsTimestampValid(const Timestamp& timestamp) { + return timestamp.seconds() <= kTimestampMaxSeconds && + timestamp.seconds() >= kTimestampMinSeconds && + timestamp.nanos() <= kTimestampMaxNanoseconds && + timestamp.nanos() >= kTimestampMinNanoseconds; + } + + static bool IsDurationValid(const Duration& duration) { + return duration.seconds() <= kDurationMaxSeconds && + duration.seconds() >= kDurationMinSeconds && + duration.nanos() <= kDurationMaxNanoseconds && + duration.nanos() >= kDurationMinNanoseconds; + } // Converts Timestamp to/from RFC 3339 date string format. // Generated output will always be Z-normalized and uses 3, 6 or 9 diff --git a/src/google/protobuf/util/time_util_test.cc b/src/google/protobuf/util/time_util_test.cc index 79e2427da..a741218c9 100644 --- a/src/google/protobuf/util/time_util_test.cc +++ b/src/google/protobuf/util/time_util_test.cc @@ -378,6 +378,138 @@ TEST(TimeUtilTest, TimestampOperators) { EXPECT_TRUE(t2 != t1); } +TEST(TimeUtilTest, IsDurationValid) { + Duration valid; + Duration overflow; + overflow.set_seconds(TimeUtil::kDurationMaxSeconds + 1); + Duration underflow; + underflow.set_seconds(TimeUtil::kDurationMinSeconds - 1); + Duration overflow_nanos; + overflow_nanos.set_nanos(TimeUtil::kDurationMaxNanoseconds + 1); + Duration underflow_nanos; + underflow_nanos.set_nanos(TimeUtil::kDurationMinNanoseconds - 1); + + EXPECT_TRUE(TimeUtil::IsDurationValid(valid)); + EXPECT_FALSE(TimeUtil::IsDurationValid(overflow)); + EXPECT_FALSE(TimeUtil::IsDurationValid(underflow)); + EXPECT_FALSE(TimeUtil::IsDurationValid(overflow_nanos)); + EXPECT_FALSE(TimeUtil::IsDurationValid(underflow_nanos)); +} + +TEST(TimeUtilTest, IsTimestampValid) { + Timestamp valid; + Timestamp overflow; + overflow.set_seconds(TimeUtil::kTimestampMaxSeconds + 1); + Timestamp underflow; + underflow.set_seconds(TimeUtil::kTimestampMinSeconds - 1); + Timestamp overflow_nanos; + overflow_nanos.set_nanos(TimeUtil::kTimestampMaxNanoseconds + 1); + Timestamp underflow_nanos; + underflow_nanos.set_nanos(TimeUtil::kTimestampMinNanoseconds - 1); + + EXPECT_TRUE(TimeUtil::IsTimestampValid(valid)); + EXPECT_FALSE(TimeUtil::IsTimestampValid(overflow)); + EXPECT_FALSE(TimeUtil::IsTimestampValid(underflow)); + EXPECT_FALSE(TimeUtil::IsTimestampValid(overflow_nanos)); + EXPECT_FALSE(TimeUtil::IsTimestampValid(underflow_nanos)); +} + +#ifdef PROTOBUF_HAS_DEATH_TEST // death tests do not work on Windows yet. + +TEST(TimeUtilTest, DurationBounds) { + Duration overflow; + overflow.set_seconds(TimeUtil::kDurationMaxSeconds + 1); + Duration underflow; + underflow.set_seconds(TimeUtil::kDurationMinSeconds - 1); + Duration overflow_nanos; + overflow_nanos.set_nanos(TimeUtil::kDurationMaxNanoseconds + 1); + Duration underflow_nanos; + underflow_nanos.set_nanos(TimeUtil::kDurationMinNanoseconds - 1); + + EXPECT_DEBUG_DEATH({ TimeUtil::SecondsToDuration(overflow.seconds()); }, + "Duration seconds"); + EXPECT_DEBUG_DEATH({ TimeUtil::SecondsToDuration(underflow.seconds()); }, + "Duration seconds"); + EXPECT_DEBUG_DEATH( + { TimeUtil::MinutesToDuration(overflow.seconds() / 60 + 1); }, + "Duration minutes"); + EXPECT_DEBUG_DEATH( + { TimeUtil::MinutesToDuration(underflow.seconds() / 60 - 1); }, + "Duration minutes"); + EXPECT_DEBUG_DEATH( + { TimeUtil::HoursToDuration(overflow.seconds() / 60 + 1); }, + "Duration hours"); + EXPECT_DEBUG_DEATH( + { TimeUtil::HoursToDuration(underflow.seconds() / 60 - 1); }, + "Duration hours"); + + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToNanoseconds(underflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::DurationToSeconds(underflow_nanos); }, + "outside of the valid range"); +} + +TEST(TimeUtilTest, TimestampBounds) { + Timestamp overflow; + overflow.set_seconds(TimeUtil::kDurationMaxSeconds + 1); + Timestamp underflow; + underflow.set_seconds(TimeUtil::kDurationMinSeconds - 1); + Timestamp overflow_nanos; + overflow_nanos.set_nanos(TimeUtil::kDurationMaxNanoseconds + 1); + Timestamp underflow_nanos; + underflow_nanos.set_nanos(TimeUtil::kDurationMinNanoseconds - 1); + + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToNanoseconds(underflow_nanos); }, + "outside of the valid range"); + + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMicroseconds(underflow_nanos); }, + "outside of the valid range"); + + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToMilliseconds(underflow_nanos); }, + "outside of the valid range"); + + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(overflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(underflow); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(overflow_nanos); }, + "outside of the valid range"); + EXPECT_DEBUG_DEATH({ TimeUtil::TimestampToSeconds(underflow_nanos); }, + "outside of the valid range"); +} + +#endif // PROTOBUF_HAS_DEATH_TEST + } // namespace } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/type_resolver_util_test.cc b/src/google/protobuf/util/type_resolver_util_test.cc index 2780a3937..f7b29889b 100644 --- a/src/google/protobuf/util/type_resolver_util_test.cc +++ b/src/google/protobuf/util/type_resolver_util_test.cc @@ -42,10 +42,10 @@ #include #include #include -#include #include #include #include +#include namespace google { namespace protobuf { diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index 2efd6cd85..538dba60e 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -8,12 +8,12 @@ #include #include -#if PROTOBUF_VERSION < 3020000 +#if PROTOBUF_VERSION < 3021000 #error This file was generated by a newer version of protoc which is #error incompatible with your Protocol Buffer headers. Please update #error your headers. #endif -#if 3020001 < PROTOBUF_MIN_PROTOC_VERSION +#if 3021000 < PROTOBUF_MIN_PROTOC_VERSION #error This file was generated by an older version of protoc which is #error incompatible with your Protocol Buffer headers. Please #error regenerate this file with a newer version of protoc. diff --git a/tests.sh b/tests.sh index e82265c89..f2b481cc7 100755 --- a/tests.sh +++ b/tests.sh @@ -246,8 +246,8 @@ build_java() { cd ../.. } -# The conformance tests are hard-coded to work with the $ROOT/java directory. -# So this can't run in parallel with two different sets of tests. +# The conformance tests are hard-coded to work with the $ROOT/java directory +# so this can't run in parallel with two different sets of tests. build_java_with_conformance_tests() { # Java build needs `protoc`. internal_build_cpp @@ -257,7 +257,7 @@ build_java_with_conformance_tests() { cd java/core && $MVN test && $MVN install cd ../lite && $MVN test && $MVN install cd ../util && $MVN test && $MVN install && $MVN package assembly:single - if [ "$version" == "jdk8" ]; then + if [ "$version" != "jdk7" ]; then cd ../kotlin && $MVN test && $MVN install cd ../kotlin-lite && $MVN test && $MVN install fi diff --git a/third_party/BUILD.bazel b/third_party/BUILD.bazel index a8b35efce..d12b3e37f 100644 --- a/third_party/BUILD.bazel +++ b/third_party/BUILD.bazel @@ -1 +1,4 @@ -exports_files(["zlib.BUILD"]) +exports_files([ + "BUILD.bazel", + "zlib.BUILD", +]) diff --git a/toolchain/cc_toolchain_config.bzl b/toolchain/cc_toolchain_config.bzl index 777bb82f9..259947777 100644 --- a/toolchain/cc_toolchain_config.bzl +++ b/toolchain/cc_toolchain_config.bzl @@ -6,6 +6,7 @@ load( "flag_group", "flag_set", "tool_path", + "with_feature_set", ) all_link_actions = [ @@ -154,10 +155,31 @@ def _impl(ctx): ), ], ), + flag_set( + actions = all_compile_actions, + flag_groups = [flag_group(flags = ["-DNDEBUG", "-O3"])], + with_features = [with_feature_set(features = ["opt"])], + ), + flag_set( + actions = all_compile_actions, + flag_groups = [flag_group(flags = ["-g"])], + with_features = [with_feature_set(features = ["dbg"])], + ), + flag_set( + actions = all_compile_actions, + flag_groups = [flag_group(flags = ["-O1"])], + with_features = [with_feature_set(features = ["fastbuild"])], + ), ], ) - features = [linker_flags, compiler_flags, sysroot_flags] + features = [ + linker_flags, + compiler_flags, + sysroot_flags, + feature(name = "dbg"), + feature(name = "opt"), + ] if "mingw" in ctx.attr.target_full_name: features.append( diff --git a/toolchain/toolchains.bazelrc b/toolchain/toolchains.bazelrc index 33b8ff947..fbd655b27 100644 --- a/toolchain/toolchains.bazelrc +++ b/toolchain/toolchains.bazelrc @@ -6,7 +6,7 @@ build:linux-ppcle_64 --config=cross_config --cpu=linux-ppcle_64 build:linux-s390_64 --config=cross_config --cpu=linux-s390_64 build:linux-x86_32 --config=cross_config --cpu=linux-x86_32 build:linux-x86_64 --config=cross_config --cpu=linux-x86_64 -build:osx-aarch_64 --config=cross_config --cpu=osx-aarch_64 -build:osx-x86_64 --config=cross_config --cpu=osx-x86_64 +build:osx-aarch_64 --config=cross_config --action_env=MACOSX_DEPLOYMENT_TARGET=10.9 --cpu=osx-aarch_64 +build:osx-x86_64 --config=cross_config --action_env=MACOSX_DEPLOYMENT_TARGET=10.9 --cpu=osx-x86_64 build:win32 --config=cross_config --cpu=win32 build:win64 --config=cross_config --cpu=win64 diff --git a/update_version.py b/update_version.py deleted file mode 100755 index 0ab9cc098..000000000 --- a/update_version.py +++ /dev/null @@ -1,405 +0,0 @@ -#!/usr/bin/env python3 -# Usage: ./update_version.py .. [] -# -# Example: -# ./update_version.py 3.7.1 2 -# => Version will become 3.7.1-rc-2 (beta) -# ./update_version.py 3.7.1 -# => Version will become 3.7.1 (stable) - -import datetime -import re -import sys -from xml.dom import minidom - -if len(sys.argv) < 2 or len(sys.argv) > 3: - print(""" -[ERROR] Please specify a version. - -./update_version.py .. [] - -Example: -./update_version.py 3.7.1 2 -""") - exit(1) - -NEW_VERSION = sys.argv[1] -NEW_VERSION_INFO = [int(x) for x in NEW_VERSION.split('.')] -if len(NEW_VERSION_INFO) != 3: - print(""" -[ERROR] Version must be in the format .. - -Example: -./update_version.py 3.7.3 -""") - exit(1) - -RC_VERSION = -1 -if len(sys.argv) > 2: - RC_VERSION = int(sys.argv[2]) - - -def Find(elem, tagname): - for child in elem.childNodes: - if child.nodeName == tagname: - return child - return None - - -def FindAndClone(elem, tagname): - return Find(elem, tagname).cloneNode(True) - - -def ReplaceText(elem, text): - elem.firstChild.replaceWholeText(text) - - -def GetFullVersion(rc_suffix = '-rc-'): - if RC_VERSION < 0: - return NEW_VERSION - else: - return '%s%s%s' % (NEW_VERSION, rc_suffix, RC_VERSION) - - -def GetSharedObjectVersion(): - protobuf_version_offset = 11 - expected_major_version = 3 - if NEW_VERSION_INFO[0] != expected_major_version: - print("""[ERROR] Major protobuf version has changed. Please update -update_version.py to readjust the protobuf_version_offset and -expected_major_version such that the PROTOBUF_VERSION in src/Makefile.am is -always increasing. - """) - exit(1) - return [NEW_VERSION_INFO[1] + protobuf_version_offset, NEW_VERSION_INFO[2], 0] - - -def RewriteXml(filename, rewriter, add_xml_prefix=True): - document = minidom.parse(filename) - rewriter(document) - # document.toxml() always prepend the XML version without inserting new line. - # We wants to preserve as much of the original formatting as possible, so we - # will remove the default XML version and replace it with our custom one when - # whever necessary. - content = document.toxml().replace('', '') - file_handle = open(filename, 'wb') - if add_xml_prefix: - file_handle.write(b'\n') - file_handle.write(content.encode('utf-8')) - file_handle.write(b'\n') - file_handle.close() - - -def RewriteTextFile(filename, line_rewriter): - lines = open(filename, 'r').readlines() - updated_lines = [] - for line in lines: - updated_lines.append(line_rewriter(line)) - if lines == updated_lines: - print('%s was not updated. Please double check.' % filename) - f = open(filename, 'w') - f.write(''.join(updated_lines)) - f.close() - - -def UpdateCMake(): - cmake_files = ( - 'cmake/libprotobuf.cmake', - 'cmake/libprotobuf-lite.cmake', - 'cmake/libprotoc.cmake' - ) - for cmake_file in cmake_files: - RewriteTextFile(cmake_file, - lambda line : re.sub( - r'SOVERSION ([0-9]+)$', - 'SOVERSION %s' % GetSharedObjectVersion()[0], - line)) - - -def UpdateConfigure(): - RewriteTextFile('configure.ac', - lambda line : re.sub( - r'^AC_INIT\(\[Protocol Buffers\],\[.*\],\[protobuf@googlegroups.com\],\[protobuf\]\)$', - ('AC_INIT([Protocol Buffers],[%s],[protobuf@googlegroups.com],[protobuf])' - % GetFullVersion()), - line)) - - -def UpdateCpp(): - cpp_version = '%d%03d%03d' % ( - NEW_VERSION_INFO[0], NEW_VERSION_INFO[1], NEW_VERSION_INFO[2]) - version_suffix = '' - if RC_VERSION != -1: - version_suffix = '-rc%s' % RC_VERSION - def RewriteCommon(line): - line = re.sub( - r'^#define GOOGLE_PROTOBUF_VERSION .*$', - '#define GOOGLE_PROTOBUF_VERSION %s' % cpp_version, - line) - line = re.sub( - r'^#define PROTOBUF_VERSION .*$', - '#define PROTOBUF_VERSION %s' % cpp_version, - line) - line = re.sub( - r'^#define GOOGLE_PROTOBUF_VERSION_SUFFIX .*$', - '#define GOOGLE_PROTOBUF_VERSION_SUFFIX "%s"' % version_suffix, - line) - line = re.sub( - r'^#define PROTOBUF_VERSION_SUFFIX .*$', - '#define PROTOBUF_VERSION_SUFFIX "%s"' % version_suffix, - line) - if NEW_VERSION_INFO[2] == 0: - line = re.sub( - r'^#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC .*$', - '#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC %s' % cpp_version, - line) - line = re.sub( - r'^#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION .*$', - '#define GOOGLE_PROTOBUF_MIN_PROTOC_VERSION %s' % cpp_version, - line) - line = re.sub( - r'^static const int kMinHeaderVersionForLibrary = .*$', - 'static const int kMinHeaderVersionForLibrary = %s;' % cpp_version, - line) - line = re.sub( - r'^static const int kMinHeaderVersionForProtoc = .*$', - 'static const int kMinHeaderVersionForProtoc = %s;' % cpp_version, - line) - return line - - def RewritePortDef(line): - line = re.sub( - r'^#define PROTOBUF_VERSION .*$', - '#define PROTOBUF_VERSION %s' % cpp_version, - line) - line = re.sub( - r'^#define PROTOBUF_VERSION_SUFFIX .*$', - '#define PROTOBUF_VERSION_SUFFIX "%s"' % version_suffix, - line) - if NEW_VERSION_INFO[2] == 0: - line = re.sub( - r'^#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC .*$', - '#define PROTOBUF_MIN_HEADER_VERSION_FOR_PROTOC %s' % cpp_version, - line) - line = re.sub( - r'^#define PROTOBUF_MIN_PROTOC_VERSION .*$', - '#define PROTOBUF_MIN_PROTOC_VERSION %s' % cpp_version, - line) - line = re.sub( - r'^#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION .*$', - '#define GOOGLE_PROTOBUF_MIN_LIBRARY_VERSION %s' % cpp_version, - line) - return line - - def RewritePbH(line): - line = re.sub( - r'^#if PROTOBUF_VERSION < .*$', - '#if PROTOBUF_VERSION < %s' % cpp_version, - line) - line = re.sub( - r'^#if .* < PROTOBUF_MIN_PROTOC_VERSION$', - '#if %s < PROTOBUF_MIN_PROTOC_VERSION' % cpp_version, - line) - return line - - RewriteTextFile('src/google/protobuf/stubs/common.h', RewriteCommon) - RewriteTextFile('src/google/protobuf/port_def.inc', RewritePortDef) - RewriteTextFile('src/google/protobuf/any.pb.h', RewritePbH) - RewriteTextFile('src/google/protobuf/api.pb.h', RewritePbH) - RewriteTextFile('src/google/protobuf/descriptor.pb.h', RewritePbH) - RewriteTextFile('src/google/protobuf/duration.pb.h', RewritePbH) - RewriteTextFile('src/google/protobuf/empty.pb.h', RewritePbH) - RewriteTextFile('src/google/protobuf/field_mask.pb.h', RewritePbH) - RewriteTextFile('src/google/protobuf/source_context.pb.h', RewritePbH) - RewriteTextFile('src/google/protobuf/struct.pb.h', RewritePbH) - RewriteTextFile('src/google/protobuf/timestamp.pb.h', RewritePbH) - RewriteTextFile('src/google/protobuf/type.pb.h', RewritePbH) - RewriteTextFile('src/google/protobuf/wrappers.pb.h', RewritePbH) - RewriteTextFile('src/google/protobuf/compiler/plugin.pb.h', RewritePbH) - - -def UpdateCsharp(): - RewriteXml('csharp/src/Google.Protobuf/Google.Protobuf.csproj', - lambda document : ReplaceText( - Find(Find(document.documentElement, 'PropertyGroup'), 'VersionPrefix'), - GetFullVersion(rc_suffix = '-rc')), - add_xml_prefix=False) - - RewriteXml('csharp/Google.Protobuf.Tools.nuspec', - lambda document : ReplaceText( - Find(Find(document.documentElement, 'metadata'), 'version'), - GetFullVersion(rc_suffix = '-rc'))) - - -def UpdateJava(): - RewriteXml('java/pom.xml', - lambda document : ReplaceText( - Find(document.documentElement, 'version'), GetFullVersion())) - - RewriteXml('java/bom/pom.xml', - lambda document : ReplaceText( - Find(document.documentElement, 'version'), GetFullVersion())) - - RewriteXml('java/core/pom.xml', - lambda document : ReplaceText( - Find(Find(document.documentElement, 'parent'), 'version'), - GetFullVersion())) - - RewriteXml('java/lite/pom.xml', - lambda document : ReplaceText( - Find(Find(document.documentElement, 'parent'), 'version'), - GetFullVersion())) - - RewriteXml('java/util/pom.xml', - lambda document : ReplaceText( - Find(Find(document.documentElement, 'parent'), 'version'), - GetFullVersion())) - - RewriteXml('java/kotlin/pom.xml', - lambda document : ReplaceText( - Find(Find(document.documentElement, 'parent'), 'version'), - GetFullVersion())) - - RewriteXml('java/kotlin-lite/pom.xml', - lambda document : ReplaceText( - Find(Find(document.documentElement, 'parent'), 'version'), - GetFullVersion())) - - RewriteXml('protoc-artifacts/pom.xml', - lambda document : ReplaceText( - Find(document.documentElement, 'version'), GetFullVersion())) - - RewriteTextFile('java/README.md', - lambda line : re.sub( - r'.*', - '%s' % GetFullVersion(), - line)) - - RewriteTextFile('java/README.md', - lambda line : re.sub( - r'implementation \'com.google.protobuf:protobuf-java:.*\'', - 'implementation \'com.google.protobuf:protobuf-java:%s\'' % GetFullVersion(), - line)) - - RewriteTextFile('java/lite.md', - lambda line : re.sub( - r'.*', - '%s' % GetFullVersion(), - line)) - - -def UpdateMakefile(): - RewriteTextFile('src/Makefile.am', - lambda line : re.sub( - r'^PROTOBUF_VERSION = .*$', - 'PROTOBUF_VERSION = %s' % ":".join(map(str,GetSharedObjectVersion())), - line)) - - -def UpdateObjectiveC(): - RewriteTextFile('Protobuf.podspec', - lambda line : re.sub( - r"^ s.version = '.*'$", - " s.version = '%s'" % GetFullVersion(rc_suffix = '-rc'), - line)) - RewriteTextFile('Protobuf-C++.podspec', - lambda line : re.sub( - r"^ s.version = '.*'$", - " s.version = '%s'" % GetFullVersion(rc_suffix = '-rc'), - line)) - - -def UpdatePhp(): - def Callback(document): - def CreateNode(tagname, indent, children): - elem = document.createElement(tagname) - indent += 1 - for child in children: - elem.appendChild(document.createTextNode('\n' + (' ' * indent))) - elem.appendChild(child) - indent -= 1 - elem.appendChild(document.createTextNode('\n' + (' ' * indent))) - return elem - - root = document.documentElement - now = datetime.datetime.now() - ReplaceText(Find(root, 'date'), now.strftime('%Y-%m-%d')) - ReplaceText(Find(root, 'time'), now.strftime('%H:%M:%S')) - version = Find(root, 'version') - ReplaceText(Find(version, 'release'), GetFullVersion(rc_suffix = 'RC')) - ReplaceText(Find(version, 'api'), NEW_VERSION) - stability = Find(root, 'stability') - ReplaceText(Find(stability, 'release'), - 'stable' if RC_VERSION < 0 else 'beta') - ReplaceText(Find(stability, 'api'), 'stable' if RC_VERSION < 0 else 'beta') - changelog = Find(root, 'changelog') - for old_version in changelog.getElementsByTagName('version'): - if Find(old_version, 'release').firstChild.nodeValue == NEW_VERSION: - print ('[WARNING] Version %s already exists in the change log.' - % NEW_VERSION) - return - if RC_VERSION != 0: - changelog.appendChild(document.createTextNode(' ')) - release = CreateNode('release', 2, [ - CreateNode('version', 3, [ - FindAndClone(version, 'release'), - FindAndClone(version, 'api') - ]), - CreateNode('stability', 3, [ - FindAndClone(stability, 'release'), - FindAndClone(stability, 'api') - ]), - FindAndClone(root, 'date'), - FindAndClone(root, 'time'), - FindAndClone(root, 'license'), - CreateNode('notes', 3, []), - ]) - changelog.appendChild(release) - changelog.appendChild(document.createTextNode('\n ')) - RewriteXml('php/ext/google/protobuf/package.xml', Callback) - RewriteTextFile('php/ext/google/protobuf/protobuf.h', - lambda line : re.sub( - r"^#define PHP_PROTOBUF_VERSION .*$", - "#define PHP_PROTOBUF_VERSION \"%s\"" % GetFullVersion(rc_suffix = 'RC'), - line)) - -def UpdatePython(): - RewriteTextFile('python/google/protobuf/__init__.py', - lambda line : re.sub( - r"^__version__ = '.*'$", - "__version__ = '%s'" % GetFullVersion(rc_suffix = 'rc'), - line)) - -def UpdateRuby(): - RewriteXml('ruby/pom.xml', - lambda document : ReplaceText( - Find(document.documentElement, 'version'), GetFullVersion())) - RewriteXml('ruby/pom.xml', - lambda document : ReplaceText( - Find(Find(Find(document.documentElement, 'dependencies'), 'dependency'), 'version'), - GetFullVersion())) - RewriteTextFile('ruby/google-protobuf.gemspec', - lambda line : re.sub( - r'^ s.version = ".*"$', - ' s.version = "%s"' % GetFullVersion(rc_suffix = '.rc.'), - line)) - -def UpdateBazel(): - RewriteTextFile('protobuf_version.bzl', - lambda line : re.sub( - r"^PROTOBUF_VERSION = '.*'$", - "PROTOBUF_VERSION = '%s'" % GetFullVersion(), - line)) - - -UpdateCMake() -UpdateConfigure() -UpdateCsharp() -UpdateCpp() -UpdateJava() -UpdateMakefile() -UpdateObjectiveC() -UpdatePhp() -UpdatePython() -UpdateRuby() -UpdateBazel()