Merge pull request #9831 from haberman/sync-stage

Integrate from Piper for C++, Java, and Python

CHANGES.txt

@@ -1,3 +1,8 @@
+Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
+
+  Compiler
+  * Require package names to be less than 512 bytes in length
+
 2022-04-05 version 3.20.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
 
   PHP

java/core/src/main/java/com/google/protobuf/TextFormat.java

@@ -43,7 +43,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

java/core/src/main/java/com/google/protobuf/Utf8.java

@@ -1611,14 +1611,14 @@ final class Utf8 {
       // ANDing the index with 7 to determine the number of bytes that need to be read before
       // we're 8-byte aligned.
       final int unaligned = 8 - ((int) offset & 7);
-      for (int j = unaligned; j > 0; j--) {
+      int i;
+      for (i = 0; i < unaligned; i++) {
         if (UnsafeUtil.getByte(bytes, offset++) < 0) {
-          return unaligned - j;
+          return i;
         }
       }
 
-      int i;
+      for (; i + 8 <= maxChars; i += 8) {
-      for (i = 0; i + 8 <= maxChars; i += 8) {
         if ((UnsafeUtil.getLong(bytes, UnsafeUtil.BYTE_ARRAY_BASE_OFFSET + offset)
                 & ASCII_MASK_LONG)
             != 0L) {

java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java

@@ -59,7 +59,6 @@ import protobuf_unittest.UnittestProto.TestAllExtensions;
 import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestAllTypes.NestedMessage;
 import protobuf_unittest.UnittestProto.TestAllTypesOrBuilder;
-import protobuf_unittest.UnittestProto.TestChildExtension;
 import protobuf_unittest.UnittestProto.TestExtremeDefaultValues;
 import protobuf_unittest.UnittestProto.TestOneof2;
 import protobuf_unittest.UnittestProto.TestPackedTypes;
@@ -2024,5 +2023,4 @@ public class GeneratedMessageTest {
     assertThat(builder.getRepeatedField(REPEATED_NESTED_MESSAGE_EXTENSION, 0))
         .isEqualTo(NestedMessage.newBuilder().setBb(100).build());
   }
-
 }

python/google/protobuf/internal/api_implementation.py

@@ -36,39 +36,40 @@ import os
 import sys
 import warnings
 
+
+def _ApiVersionToImplementationType(api_version):
+  if api_version == 2:
+    return 'cpp'
+  if api_version == 1:
+    raise ValueError('api_version=1 is no longer supported.')
+  if api_version == 0:
+    return 'python'
+  return None
+
+
+_implementation_type = None
 try:
   # pylint: disable=g-import-not-at-top
   from google.protobuf.internal import _api_implementation
   # The compile-time constants in the _api_implementation module can be used to
   # switch to a certain implementation of the Python API at build time.
-  _api_version = _api_implementation.api_version
+  _implementation_type = _ApiVersionToImplementationType(
+      _api_implementation.api_version)
 except ImportError:
-  _api_version = -1  # Unspecified by compiler flags.
+  pass  # Unspecified by compiler flags.
-
-if _api_version == 1:
-  raise ValueError('api_version=1 is no longer supported.')
 
 
 
 
-def _ApiVersionToImplementationType(api_version):
+if _implementation_type is None:
-  if api_version == 3:
+  _implementation_type = 'python'
-    return 'upb'
-  if api_version == 2:
-    return 'cpp'
-  return 'python'
-
-# TODO(jieluo): Remove _api_version and only keep implementation_type
-# http://b/228103078
-_default_implementation_type = _ApiVersionToImplementationType(_api_version)
-
 
 # This environment variable can be used to switch to a certain implementation
 # of the Python API, overriding the compile-time constants in the
 # _api_implementation module. Right now only 'python', 'cpp' and 'upb' are
 # valid values. Any other value will raise error.
 _implementation_type = os.getenv('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION',
-                                 _default_implementation_type)
+                                 _implementation_type)
 
 if _implementation_type not in ('python', 'cpp', 'upb'):
   raise ValueError('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION {0} is not '

python/google/protobuf/internal/text_format_test.py

@@ -40,6 +40,7 @@ import textwrap
 import unittest
 
 from google.protobuf import any_pb2
+from google.protobuf import struct_pb2
 from google.protobuf import any_test_pb2
 from google.protobuf import map_unittest_pb2
 from google.protobuf import unittest_custom_options_pb2
@@ -1939,6 +1940,16 @@ class Proto3Tests(unittest.TestCase):
       text_format.Merge(text, message)
     self.assertEqual(str(e.exception), '3:11 : Expected "}".')
 
+  def testParseExpandedAnyListValue(self):
+    any_msg = any_pb2.Any()
+    any_msg.Pack(struct_pb2.ListValue())
+    msg = any_test_pb2.TestAny(any_value=any_msg)
+    text = ('any_value {\n'
+            '  [type.googleapis.com/google.protobuf.ListValue] {}\n'
+            '}\n')
+    parsed_msg = text_format.Parse(text, any_test_pb2.TestAny())
+    self.assertEqual(msg, parsed_msg)
+
   def testProto3Optional(self):
     msg = test_proto3_optional_pb2.TestProto3Optional()
     self.assertEqual(text_format.MessageToString(msg), '')

python/google/protobuf/internal/unknown_fields_test.py

@@ -365,7 +365,7 @@ class UnknownFieldsAccessorsTest(unittest.TestCase):
   def testUnknownExtensions(self):
     message = unittest_pb2.TestEmptyMessageWithExtensions()
     message.ParseFromString(self.all_fields_data)
-    self.assertEqual(len(message.UnknownFields()), 98)
+    self.assertEqual(len(unknown_fields.UnknownFieldSet(message)), 98)
     self.assertEqual(message.SerializeToString(), self.all_fields_data)
 
 

python/google/protobuf/text_format.py

@@ -886,7 +886,10 @@ class _Parser(object):
         expanded_any_end_token = '}'
       expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name,
                                                            self.descriptor_pool)
-      if not expanded_any_sub_message:
+      # Direct comparison with None is used instead of implicit bool conversion
+      # to avoid false positives with falsy initial values, e.g. for
+      # google.protobuf.ListValue.
+      if expanded_any_sub_message is None:
         raise ParseError('Type %s not found in descriptor pool' %
                          packed_type_name)
       while not tokenizer.TryConsume(expanded_any_end_token):

python/google/protobuf/unknown_fields.py

@@ -40,17 +40,13 @@ Simple usage example:
 
 
 from google.protobuf.internal import api_implementation
-if api_implementation.Type() != 'python':
+
-  from google.protobuf.pyext import _message  # pylint: disable=g-import-not-at-top
+if api_implementation._c_module is not None:  # pylint: disable=protected-access
+  UnknownFieldSet = api_implementation._c_module.UnknownFieldSet  # pylint: disable=protected-access
 else:
   from google.protobuf.internal import decoder  # pylint: disable=g-import-not-at-top
   from google.protobuf.internal import wire_format  # pylint: disable=g-import-not-at-top
 
-if api_implementation.Type() == 'cpp':
-
-  UnknownFieldSet = _message.UnknownFieldSet
-
-else:
   class UnknownField:
     """A parsed unknown field."""
 

src/google/protobuf/compiler/cpp/helpers.cc

@@ -193,9 +193,11 @@ bool IsLazy(const FieldDescriptor* field, const Options& options,
          IsEagerlyVerifiedLazy(field, options, scc_analyzer);
 }
 
-bool IsEagerlyVerifiedLazyByProfile(const FieldDescriptor* field,
+// Returns true if "field" is a message field that is backed by LazyField per
-                                    const Options& options,
+// profile (go/pdlazy).
-                                    MessageSCCAnalyzer* scc_analyzer) {
+inline bool IsEagerlyVerifiedLazyByProfile(const FieldDescriptor* field,
+                                           const Options& options,
+                                           MessageSCCAnalyzer* scc_analyzer) {
   return false;
 }
 

src/google/protobuf/compiler/cpp/helpers.h

@@ -364,12 +364,6 @@ inline bool IsExplicitLazy(const FieldDescriptor* field) {
   return field->options().lazy() || field->options().unverified_lazy();
 }
 
-// Returns true if "field" is a message field that is backed by LazyField per
-// profile (go/pdlazy).
-bool IsEagerlyVerifiedLazyByProfile(const FieldDescriptor* field,
-                                    const Options& options,
-                                    MessageSCCAnalyzer* scc_analyzer);
-
 bool IsEagerlyVerifiedLazy(const FieldDescriptor* field, const Options& options,
                            MessageSCCAnalyzer* scc_analyzer);
 

src/google/protobuf/compiler/cpp/message.cc

@@ -2250,7 +2250,7 @@ std::pair<size_t, size_t> MessageGenerator::GenerateOffsets(
     //
     // Embed whether the field is eagerly verified lazy or inlined string to the
     // LSB of the offset.
-    if (IsEagerlyVerifiedLazyByProfile(field, options_, scc_analyzer_)) {
+    if (IsEagerlyVerifiedLazy(field, options_, scc_analyzer_)) {
       format(" | 0x1u  // eagerly verified lazy\n");
     } else if (IsStringInlined(field, options_)) {
       format(" | 0x1u  // inlined\n");

src/google/protobuf/descriptor.cc

@@ -5031,6 +5031,13 @@ const FileDescriptor* DescriptorBuilder::BuildFile(
     }
   }
 
+  static const int kMaximumPackageLength = 511;
+  if (proto.package().size() > kMaximumPackageLength) {
+    AddError(proto.package(), proto, DescriptorPool::ErrorCollector::NAME,
+             "Package name is too long");
+    return nullptr;
+  }
+
   // If we have a fallback_database_, and we aren't doing lazy import building,
   // attempt to load all dependencies now, before checkpointing tables_.  This
   // avoids confusion with recursive checkpoints.

src/google/protobuf/descriptor_unittest.cc

@@ -6865,6 +6865,30 @@ TEST_F(ValidationErrorTest, UnusedImportWithOtherError) {
 }
 
 
+TEST_F(ValidationErrorTest, PackageTooLong) {
+  BuildFileWithErrors(
+      "name: \"foo.proto\" "
+      "syntax: \"proto3\" "
+      "package: "
+      "\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaa\"",
+      "foo.proto: "
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+      "aaaaaaaa: NAME: Package name is too long\n");
+}
+
 // ===================================================================
 // DescriptorDatabase
 

src/google/protobuf/generated_message_reflection.cc

@@ -257,7 +257,11 @@ bool Reflection::IsLazyExtension(const Message& message,
 }
 
 bool Reflection::IsLazilyVerifiedLazyField(const FieldDescriptor* field) const {
-  return field->options().lazy() || field->options().unverified_lazy();
+  if (field->options().unverified_lazy()) return true;
+
+  // Message fields with [lazy=true] will be eagerly verified
+  // (go/verified-lazy).
+  return field->options().lazy() && !IsEagerlyVerifiedLazyField(field);
 }
 
 bool Reflection::IsEagerlyVerifiedLazyField(

src/google/protobuf/generated_message_reflection_unittest.cc

@@ -78,6 +78,14 @@ class GeneratedMessageReflectionTestHelper {
   static bool IsLazyField(const Message& msg, const FieldDescriptor* field) {
     return msg.GetReflection()->IsLazyField(field);
   }
+  static bool IsEagerlyVerifiedLazyField(const Message& msg,
+                                         const FieldDescriptor* field) {
+    return msg.GetReflection()->IsEagerlyVerifiedLazyField(field);
+  }
+  static bool IsLazilyVerifiedLazyField(const Message& msg,
+                                        const FieldDescriptor* field) {
+    return msg.GetReflection()->IsLazilyVerifiedLazyField(field);
+  }
 };
 
 namespace {

src/google/protobuf/metadata_lite.h

@@ -60,7 +60,7 @@ namespace internal {
 // It uses bit 0 == 0 to indicate an arena pointer and bit 0 == 1 to indicate a
 // UFS+Arena-container pointer. Besides it uses bit 1 == 0 to indicate arena
 // allocation and bit 1 == 1 to indicate heap allocation.
-class InternalMetadata {
+class PROTOBUF_EXPORT InternalMetadata {
  public:
   constexpr InternalMetadata() : ptr_(0) {}
   explicit InternalMetadata(Arena* arena, bool is_message_owned = false) {