Sync from Piper @447000875

PROTOBUF_SYNC_PIPER

BUILD.bazel

@@ -801,10 +801,13 @@ cc_test(
         "src/google/protobuf/**/*",
         # Files for csharp_bootstrap_unittest.cc.
         "conformance/**/*",
-    ]) + glob([
-        # Files for csharp_bootstrap_unittest.cc.
-        "csharp/src/**/*",
-    ], allow_empty=True),
+    ]) + glob(
+        [
+            # Files for csharp_bootstrap_unittest.cc.
+            "csharp/src/**/*",
+        ],
+        allow_empty = True,
+    ),
     includes = [
         "src/",
     ],
@@ -1020,10 +1023,6 @@ py_proto_library(
 
 py_library(
     name = "protobuf_python",
-    deps = [
-        ":well_known_types_py_pb2",
-        ":python_srcs",
-    ],
     data = select({
         "//conditions:default": [],
         ":use_fast_cpp_protos": [
@@ -1031,6 +1030,10 @@ py_library(
             ":python/google/protobuf/pyext/_message.so",
         ],
     }),
+    deps = [
+        ":python_srcs",
+        ":well_known_types_py_pb2",
+    ],
 )
 
 # Copy the test proto files from src/google/protobuf to
@@ -1058,8 +1061,8 @@ py_proto_library(
     default_runtime = "",
     protoc = ":protoc",
     srcs_version = "PY2AND3",
-    deps = [":well_known_types_py_pb2"],
     visibility = ["//visibility:public"],
+    deps = [":well_known_types_py_pb2"],
 )
 
 py_proto_library(
@@ -1072,8 +1075,8 @@ py_proto_library(
     default_runtime = ":protobuf_python",
     protoc = ":protoc",
     srcs_version = "PY2AND3",
-    deps = [":python_common_test_protos"],
     visibility = ["//visibility:public"],
+    deps = [":python_common_test_protos"],
 )
 
 py_library(
@@ -1139,12 +1142,14 @@ proto_lang_toolchain(
 alias(
     name = "objectivec",
     actual = "//objectivec",
+    tags = ["manual"],
     visibility = ["//visibility:public"],
 )
 
 alias(
     name = "protobuf_objc",
     actual = "//objectivec",
+    tags = ["manual"],
     visibility = ["//visibility:public"],
 )
 
@@ -1155,7 +1160,7 @@ alias(
 genrule(
     name = "generated_protos",
     srcs = ["src/google/protobuf/unittest_import.proto"],
-    outs = ["unittest_gen.proto"],
+    outs = ["unittest_gen_import.proto"],
     cmd = "cat $(SRCS) | sed 's|google/|src/google/|' >  $(OUTS)",
 )
 
@@ -1474,28 +1479,31 @@ internal_gen_kt_protos(
 # Files included in all source distributions
 pkg_files(
     name = "common_dist_files",
-    srcs = glob([
-        "*.bzl",
-        "cmake/*.cmake",
-        "cmake/*.in",
-        "editors/*",
+    srcs = glob(
+        [
+            "*.bzl",
+            "cmake/*.cmake",
+            "cmake/*.in",
+            "editors/*",
 
-        # Several of these files are generated by autogen.sh, so using
-        # glob() lets us ignore them if they are missing. (This is not good
-        # practice, though.)
-        "Makefile.in",
-        "aclocal.m4",
-        "ar-lib",
-        "compile",
-        "config*",
-        "depcomp",
-        "install-sh",
-        "ltmain.sh",
-        "m4/*.m4",
-        "missing",
-        "protobuf*.pc.in",
-        "test-driver",
-    ], allow_empty = True) + [
+            # Several of these files are generated by autogen.sh, so using
+            # glob() lets us ignore them if they are missing. (This is not good
+            # practice, though.)
+            "Makefile.in",
+            "aclocal.m4",
+            "ar-lib",
+            "compile",
+            "config*",
+            "depcomp",
+            "install-sh",
+            "ltmain.sh",
+            "m4/*.m4",
+            "missing",
+            "protobuf*.pc.in",
+            "test-driver",
+        ],
+        allow_empty = True,
+    ) + [
         "BUILD.bazel",
         "CHANGES.txt",
         "CMakeLists.txt",
@@ -1569,15 +1577,6 @@ pkg_files(
     visibility = ["//pkg:__pkg__"],
 )
 
-# Additional files for PHP
-pkg_files(
-    name = "php_dist_files",
-    srcs = [
-        "composer.json",
-    ],
-    visibility = ["//pkg:__pkg__"],
-)
-
 # Python runtime
 pkg_files(
     name = "python_dist_files",

java/core/BUILD.bazel

@@ -121,6 +121,7 @@ java_export(
     resources = [
         "//:lite_well_known_protos",
     ],
+    tags = ["manual"],
     runtime_deps = [":lite"],
 )
 
@@ -156,6 +157,7 @@ java_export(
     resources = [
         "//:well_known_protos",
     ],
+    tags = ["manual"],
     runtime_deps = [":core"],
 )
 

java/kotlin-lite/BUILD.bazel

@@ -37,6 +37,7 @@ kt_jvm_export(
     maven_coordinates = "com.google.protobuf:protobuf-kotlin-lite:%s" % PROTOBUF_JAVA_VERSION,
     pom_template = "//java/kotlin-lite:pom_template.xml",
     resources = ["//:well_known_protos"],
+    tags = ["manual"],
     runtime_deps = [
         ":lite_extensions",
         ":well_known_protos_kotlin_lite",

java/kotlin/BUILD.bazel

@@ -59,6 +59,7 @@ kt_jvm_export(
     maven_coordinates = "com.google.protobuf:protobuf-kotlin:%s" % PROTOBUF_JAVA_VERSION,
     pom_template = "//java/kotlin:pom_template.xml",
     resources = ["//:well_known_protos"],
+    tags = ["manual"],
     runtime_deps = [
         ":bytestring_lib",
         ":full_extensions",

java/util/BUILD.bazel

@@ -27,6 +27,7 @@ java_export(
     deploy_env = ["//java/core"],
     maven_coordinates = "com.google.protobuf:protobuf-java-util:%s" % PROTOBUF_JAVA_VERSION,
     pom_template = "pom_template.xml",
+    tags = ["manual"],
     visibility = ["//java:__pkg__"],
     runtime_deps = [":util"],
 )

pkg/BUILD.bazel

@@ -211,7 +211,6 @@ pkg_filegroup(
     name = "php_srcs",
     srcs = [
         ":dist_common",
-        "//:php_dist_files",
         "//php:dist_files",
     ],
 )
@@ -339,11 +338,9 @@ gen_automake_file_lists(
         "//csharp:dist_files": "dist_csharp2",
         "//:objectivec_dist_files": "dist_objectivec",
         "//objectivec:dist_files": "dist_objectivec2",
-        "//:php_dist_files": "dist_php",
-        "//php:dist_files": "dist_php2",
+        "//php:dist_files": "dist_php",
         "//:python_dist_files": "dist_python",
         "//ruby:dist_files": "dist_ruby",
-        "//js:dist_files": "dist_js",
     },
 )
 

protobuf_deps.bzl

@@ -114,6 +114,6 @@ def protobuf_deps():
         _github_archive(
             name = "upb",
             repo = "https://github.com/protocolbuffers/upb",
-            commit = "2abcb7dd8a524d73322082951d334392c76721b9",
-            sha256 = "f583f1389a7531940a51221b21b69a421b4c12811a91fb5df493cd0bb13ee5d6",
+            commit = "0e8772fc20e5a0a2fa1f326c79d494374871ef94",
+            sha256 = "2a7ccd5a8e20c5b2f6efafb305f734c7ad1cc5615706f7a35cd4fbad436d5101",
         )

protobuf_version.bzl

@@ -1,2 +1,6 @@
 PROTOC_VERSION = '3.20.1'
 PROTOBUF_JAVA_VERSION = '3.20.1'
+PROTOBUF_PYTHON_VERSION = '3.20.1'
+
+# Remove once upb has been updated to use PROTOBUF_PYTHON_VERSION instead.
+PROTOBUF_VERSION = PROTOBUF_PYTHON_VERSION

src/google/protobuf/arena.h

@@ -414,6 +414,8 @@ class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
     // Provides access to protected GetOwningArena to generated messages.
     static Arena* GetOwningArena(const T* p) { return p->GetOwningArena(); }
 
+    static void InternalSwap(T* a, T* b) { a->InternalSwap(b); }
+
     static Arena* GetArenaForAllocationInternal(
         const T* p, std::true_type /*is_derived_from<MessageLite>*/) {
       return p->GetArenaForAllocation();

src/google/protobuf/compiler/cpp/helpers.cc

@@ -202,64 +202,6 @@ inline VerifySimpleType VerifyInt32TypeToVerifyCustom(VerifyInt32Type t) {
                       static_cast<int32_t>(VerifyInt32Type::kCustom)];
 }
 
-// Returns one of int32 verification types: kNever, kCustom, kAlways.
-//
-// We need to verify int32 encoding to detect non-canonical encoding (5B for
-// negative int32) and fallback to eager parsing.
-//
-// kNever skips int32 check  because there is no int32 field. kAlways
-// unconditionally verifies int32 encoding because all or almost varint fields
-// are int32. Otherwise, kCustom verifies int32 encoding only on exact field
-// number match. Note the following tweaks:
-// --uint32 very likely causes false positives. Having one requires kCustom.
-// --kCustom may be cheap enough if all int32 fields fit into a bitmask.
-// --Otherwise, try always check if X% of varint fields are int32.
-VerifyInt32Type ShouldVerifyInt32(const Descriptor* descriptor) {
-  int num_int32 = 0;
-  int num_int32_big_number = 0;
-  int num_uint32 = 0;
-  int num_other_varint = 0;
-
-  for (const auto* field : FieldRange(descriptor)) {
-    switch (field->type()) {
-      case FieldDescriptor::TYPE_INT32:
-        ++num_int32;
-        if (field->number() > 64) ++num_int32_big_number;
-        break;
-      case FieldDescriptor::TYPE_UINT32:
-        ++num_uint32;
-        ++num_other_varint;
-        break;
-      default:
-        if (internal::WireFormat::WireTypeForFieldType(field->type()) ==
-            internal::WireFormatLite::WIRETYPE_VARINT) {
-          ++num_other_varint;
-        }
-        break;
-    }
-  }
-
-  // If there is no int32 fields, no need to check int32 encoding.
-  if (num_int32 == 0) return VerifyInt32Type::kNever;
-
-  // If all varint fields are int32, *always* check int32 encoding.
-  if (num_other_varint == 0) return VerifyInt32Type::kAlways;
-
-  // Negative uint32 encoding will cause fallback eager parsing as it appears
-  // non-canonical encoding. Also, if all int32 fields fit into a 64 bit mask,
-  // checking bitmask is affordable. Try exact match in these cases.
-  if (num_uint32 > 0 || num_int32_big_number == 0) {
-    return VerifyInt32Type::kCustom;
-  }
-
-  // If a given varint is likely int32, we should just always check. Let's use
-  // an arbitrary threshold of 75% (#int32 / #varints).
-  constexpr int kLikelyInt32Pct = 75;
-  return (100 * num_int32) / (num_int32 + num_other_varint) >= kLikelyInt32Pct
-             ? VerifyInt32Type::kAlways
-             : VerifyInt32Type::kCustom;
-}
-
 }  // namespace
 
 bool IsLazy(const FieldDescriptor* field, const Options& options,
@@ -574,7 +516,7 @@ std::string FieldMemberName(const FieldDescriptor* field, bool split) {
   if (field->real_containing_oneof() == nullptr) {
     return StrCat(prefix, split_prefix, FieldName(field), "_");
   }
-  // Oneof fields are enver split.
+  // Oneof fields are never split.
   GOOGLE_CHECK(!split);
   return StrCat(prefix, field->containing_oneof()->name(), "_.",
                       FieldName(field), "_");

src/google/protobuf/compiler/cpp/options.h

@@ -80,7 +80,7 @@ struct Options {
   bool annotate_accessor = false;
   bool unused_field_stripping = false;
   bool unverified_lazy_message_sets = false;
-  bool unverified_lazy = true;
+  bool unverified_lazy = false;
   bool profile_driven_inline_string = true;
   bool message_owned_arena_trial = false;
   bool force_split = false;

src/google/protobuf/map.h

@@ -1363,11 +1363,8 @@ class Map {
   }
   template <class InputIt>
   void insert(InputIt first, InputIt last) {
-    for (InputIt it = first; it != last; ++it) {
-      iterator exist_it = find(it->first);
-      if (exist_it == end()) {
-        operator[](it->first) = it->second;
-      }
+    for (; first != last; ++first) {
+      try_emplace(first->first, first->second);
     }
   }
   void insert(std::initializer_list<value_type> values) {

src/google/protobuf/message_unittest.inc

@@ -285,6 +285,58 @@ TEST(MESSAGE_TEST_NAME, MergeFromUninitialized) {
   EXPECT_TRUE(TestUtil::EqualsToSerialized(q, p.SerializePartialAsString()));
 }
 
+TEST(MESSAGE_TEST_NAME, ExplicitLazyExceedRecursionLimit) {
+  UNITTEST::NestedTestAllTypes original, parsed;
+  // Build proto with recursion depth of 3.
+  original.mutable_lazy_child()
+      ->mutable_child()
+      ->mutable_payload()
+      ->set_optional_int32(-1);
+  std::string serialized;
+  EXPECT_TRUE(original.SerializeToString(&serialized));
+
+  // User annotated LazyField ([lazy = true]) is eagerly verified and should
+  // catch the recursion limit violation.
+  io::ArrayInputStream array_stream(serialized.data(), serialized.size());
+  io::CodedInputStream input_stream(&array_stream);
+  input_stream.SetRecursionLimit(2);
+  EXPECT_FALSE(parsed.ParseFromCodedStream(&input_stream));
+
+  // Lazy read results in parsing error which can be verified by not having
+  // expected value.
+  EXPECT_NE(parsed.lazy_child().child().payload().optional_int32(), -1);
+}
+
+TEST(MESSAGE_TEST_NAME, NestedExplicitLazyExceedRecursionLimit) {
+  UNITTEST::NestedTestAllTypes original, parsed;
+  // Build proto with recursion depth of 5, with nested annotated LazyField.
+  original.mutable_lazy_child()
+      ->mutable_child()
+      ->mutable_lazy_child()
+      ->mutable_child()
+      ->mutable_payload()
+      ->set_optional_int32(-1);
+  std::string serialized;
+  EXPECT_TRUE(original.SerializeToString(&serialized));
+
+  // User annotated LazyField ([lazy = true]) is eagerly verified and should
+  // catch the recursion limit violation.
+  io::ArrayInputStream array_stream(serialized.data(), serialized.size());
+  io::CodedInputStream input_stream(&array_stream);
+  input_stream.SetRecursionLimit(4);
+  EXPECT_FALSE(parsed.ParseFromCodedStream(&input_stream));
+
+  // Lazy read results in parsing error which can be verified by not having
+  // expected value.
+  EXPECT_NE(parsed.lazy_child()
+                .child()
+                .lazy_child()
+                .child()
+                .payload()
+                .optional_int32(),
+            -1);
+}
+
 TEST(MESSAGE_TEST_NAME, ParseFailsIfSubmessageTruncated) {
   UNITTEST::NestedTestAllTypes o, p;
   constexpr int kDepth = 5;

src/google/protobuf/port_def.inc

@@ -655,7 +655,7 @@
 #define PROTOBUF_CONSTEXPR constexpr
 #else
 #define PROTOBUF_CONSTINIT
-#define PROTOBUF_CONSTEXPR inline
+#define PROTOBUF_CONSTEXPR
 #endif
 
 // Some globals with an empty non-trivial destructor are annotated with

src/google/protobuf/repeated_field.h

@@ -346,7 +346,7 @@ class RepeatedField final {
   int total_size_;
   // Pad the Rep after arena allow for power-of-two byte sizes when
   // sizeof(Element) > sizeof(Arena*). eg for 16-byte objects.
-  static constexpr size_t kRepHeaderSize =
+  static PROTOBUF_CONSTEXPR const size_t kRepHeaderSize =
       sizeof(Arena*) < sizeof(Element) ? sizeof(Element) : sizeof(Arena*);
   struct Rep {
     Arena* arena;

version.json

@@ -10,7 +10,7 @@
             "javascript": "3.21-dev",
             "objectivec": "3.21-dev",
             "php": "3.21-dev",
-            "python": "3.21-dev",
+            "python": "4.21-dev",
             "ruby": "3.21-dev"
         }
     }