Sync from Piper @429642851

PROTOBUF_SYNC_PIPER
This commit is contained in:
David L. Jones 2022-02-18 14:02:12 -08:00
parent 2fbda76fad
commit 3ab97ce830
14 changed files with 252 additions and 91 deletions

View File

@ -87,6 +87,7 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
* Remove GetPointer() and explicit nullptr defaults.
* add proto_h flag for speeding up large builds
* Add missing overload for reference wrapped fields.
* Add MergedDescriptorDatabase::FindAllFileNames()
2022-01-28 version 3.19.4 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)

View File

@ -19,6 +19,7 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\api.pb.h" include\goo
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arena.h" include\google\protobuf\arena.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arena_impl.h" include\google\protobuf\arena_impl.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arenastring.h" include\google\protobuf\arenastring.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\arenaz_sampler.h" include\google\protobuf\arenaz_sampler.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\code_generator.h" include\google\protobuf\compiler\code_generator.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\command_line_interface.h" include\google\protobuf\compiler\command_line_interface.h
copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\cpp\cpp_file.h" include\google\protobuf\compiler\cpp\cpp_file.h

View File

@ -41,6 +41,10 @@ import com.google.protobuf.testing.Proto3Testing.Proto3MessageWithMaps;
/** Schemas to support testing. */
public class TestSchemas {
private TestSchemas() {
}
public static final Schema<Proto2Message> genericProto2Schema =
new ManifestSchemaFactory().createSchema(Proto2Message.class);
public static final Schema<Proto3Message> genericProto3Schema =

View File

@ -48,7 +48,6 @@ __author__ = 'robinson@google.com (Will Robinson)'
import ctypes
import numbers
from google.protobuf.internal import api_implementation
from google.protobuf.internal import decoder
from google.protobuf.internal import encoder
from google.protobuf.internal import wire_format
@ -77,7 +76,8 @@ def ToShortestFloat(original):
def SupportsOpenEnums(field_descriptor):
return field_descriptor.containing_type.syntax == "proto3"
return field_descriptor.containing_type.syntax == 'proto3'
def GetTypeChecker(field):
"""Returns a type checker for a message field of the specified types.
@ -105,7 +105,6 @@ def GetTypeChecker(field):
# subclassing builtin types and doing weird things. We're not trying to
# protect against malicious clients here, just people accidentally shooting
# themselves in the foot in obvious ways.
class TypeChecker(object):
"""Type checker used to catch type errors as early as possible
@ -124,11 +123,6 @@ class TypeChecker(object):
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), self._acceptable_types))
raise TypeError(message)
# Some field types(float, double and bool) accept other types, must
# convert to the correct type in such cases.
if self._acceptable_types:
if self._acceptable_types[0] in (bool, float):
return self._acceptable_types[0](proposed_value)
return proposed_value
@ -142,6 +136,22 @@ class TypeCheckerWithDefault(TypeChecker):
return self._default_value
class BoolValueChecker(object):
"""Type checker used for bool fields."""
def CheckValue(self, proposed_value):
if not hasattr(proposed_value, '__index__') or (
type(proposed_value).__module__ == 'numpy' and
type(proposed_value).__name__ == 'ndarray'):
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), (bool, int)))
raise TypeError(message)
return bool(proposed_value)
def DefaultValue(self):
return False
# IntValueChecker and its subclasses perform integer type-checks
# and bounds-checks.
class IntValueChecker(object):
@ -149,10 +159,13 @@ class IntValueChecker(object):
"""Checker used for integer fields. Performs type-check and range check."""
def CheckValue(self, proposed_value):
if not isinstance(proposed_value, numbers.Integral):
if not hasattr(proposed_value, '__index__') or (
type(proposed_value).__module__ == 'numpy' and
type(proposed_value).__name__ == 'ndarray'):
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), (int,)))
raise TypeError(message)
if not self._MIN <= int(proposed_value) <= self._MAX:
raise ValueError('Value out of range: %d' % proposed_value)
# We force all values to int to make alternate implementations where the
@ -249,20 +262,38 @@ _INF = float('inf')
_NEG_INF = float('-inf')
class FloatValueChecker(object):
class DoubleValueChecker(object):
"""Checker used for double fields.
"""Checker used for float fields. Performs type-check and range check.
Performs type-check and range check.
"""
def CheckValue(self, proposed_value):
"""Check and convert proposed_value to float."""
if (not hasattr(proposed_value, '__float__') and
not hasattr(proposed_value, '__index__')) or (
type(proposed_value).__module__ == 'numpy' and
type(proposed_value).__name__ == 'ndarray'):
message = ('%.1024r has type %s, but expected one of: int, float' %
(proposed_value, type(proposed_value)))
raise TypeError(message)
return float(proposed_value)
def DefaultValue(self):
return 0.0
class FloatValueChecker(DoubleValueChecker):
"""Checker used for float fields.
Performs type-check and range check.
Values exceeding a 32-bit float will be converted to inf/-inf.
"""
def CheckValue(self, proposed_value):
"""Check and convert proposed_value to float."""
if not isinstance(proposed_value, numbers.Real):
message = ('%.1024r has type %s, but expected one of: numbers.Real' %
(proposed_value, type(proposed_value)))
raise TypeError(message)
converted_value = float(proposed_value)
converted_value = super().CheckValue(proposed_value)
# This inf rounding matches the C++ proto SafeDoubleToFloat logic.
if converted_value > _FLOAT_MAX:
return _INF
@ -271,23 +302,17 @@ class FloatValueChecker(object):
return TruncateToFourByteFloat(converted_value)
def DefaultValue(self):
return 0.0
# Type-checkers for all scalar CPPTYPEs.
_VALUE_CHECKERS = {
_FieldDescriptor.CPPTYPE_INT32: Int32ValueChecker(),
_FieldDescriptor.CPPTYPE_INT64: Int64ValueChecker(),
_FieldDescriptor.CPPTYPE_UINT32: Uint32ValueChecker(),
_FieldDescriptor.CPPTYPE_UINT64: Uint64ValueChecker(),
_FieldDescriptor.CPPTYPE_DOUBLE: TypeCheckerWithDefault(
0.0, float, numbers.Real),
_FieldDescriptor.CPPTYPE_DOUBLE: DoubleValueChecker(),
_FieldDescriptor.CPPTYPE_FLOAT: FloatValueChecker(),
_FieldDescriptor.CPPTYPE_BOOL: TypeCheckerWithDefault(
False, bool, numbers.Integral),
_FieldDescriptor.CPPTYPE_BOOL: BoolValueChecker(),
_FieldDescriptor.CPPTYPE_STRING: TypeCheckerWithDefault(b'', bytes),
}
}
# Map from field type to a function F, such that F(field_num, value)

View File

@ -576,32 +576,23 @@ template <class T>
bool CheckAndGetInteger(PyObject* arg, T* value) {
// This effectively defines an integer as "an object that can be cast as
// an integer and can be used as an ordinal number".
// This definition includes everything that implements numbers.Integral
// This definition includes everything with a valid __index__() implementation
// and shouldn't cast the net too wide.
if (PROTOBUF_PREDICT_FALSE(!PyIndex_Check(arg))) {
FormatTypeError(arg, "int, long");
if (!strcmp(Py_TYPE(arg)->tp_name, "numpy.ndarray") ||
PROTOBUF_PREDICT_FALSE(!PyIndex_Check(arg))) {
FormatTypeError(arg, "int");
return false;
}
PyObject* arg_py_int = PyNumber_Index(arg);
if (PyErr_Occurred()) {
// Propagate existing error.
return false;
}
// Now we have an integral number so we can safely use PyLong_ functions.
// We need to treat the signed and unsigned cases differently in case arg is
// holding a value above the maximum for signed longs.
if (std::numeric_limits<T>::min() == 0) {
// Unsigned case.
unsigned PY_LONG_LONG ulong_result;
if (PyLong_Check(arg)) {
ulong_result = PyLong_AsUnsignedLongLong(arg);
} else {
// Unlike PyLong_AsLongLong, PyLong_AsUnsignedLongLong is very
// picky about the exact type.
PyObject* casted = PyNumber_Long(arg);
if (PROTOBUF_PREDICT_FALSE(casted == nullptr)) {
// Propagate existing error.
return false;
}
ulong_result = PyLong_AsUnsignedLongLong(casted);
Py_DECREF(casted);
}
unsigned PY_LONG_LONG ulong_result = PyLong_AsUnsignedLongLong(arg_py_int);
if (VerifyIntegerCastAndRange<T, unsigned PY_LONG_LONG>(arg,
ulong_result)) {
*value = static_cast<T>(ulong_result);
@ -610,30 +601,13 @@ bool CheckAndGetInteger(PyObject* arg, T* value) {
}
} else {
// Signed case.
PY_LONG_LONG long_result;
PyNumberMethods *nb;
if ((nb = arg->ob_type->tp_as_number) != nullptr && nb->nb_int != nullptr) {
// PyLong_AsLongLong requires it to be a long or to have an __int__()
// method.
long_result = PyLong_AsLongLong(arg);
} else {
// Valid subclasses of numbers.Integral should have a __long__() method
// so fall back to that.
PyObject* casted = PyNumber_Long(arg);
if (PROTOBUF_PREDICT_FALSE(casted == nullptr)) {
// Propagate existing error.
return false;
}
long_result = PyLong_AsLongLong(casted);
Py_DECREF(casted);
}
PY_LONG_LONG long_result = PyLong_AsLongLong(arg);
if (VerifyIntegerCastAndRange<T, PY_LONG_LONG>(arg, long_result)) {
*value = static_cast<T>(long_result);
} else {
return false;
}
}
return true;
}
@ -646,8 +620,9 @@ template bool CheckAndGetInteger<uint64>(PyObject*, uint64*);
bool CheckAndGetDouble(PyObject* arg, double* value) {
*value = PyFloat_AsDouble(arg);
if (PROTOBUF_PREDICT_FALSE(*value == -1 && PyErr_Occurred())) {
FormatTypeError(arg, "int, long, float");
if (!strcmp(Py_TYPE(arg)->tp_name, "numpy.ndarray") ||
PROTOBUF_PREDICT_FALSE(*value == -1 && PyErr_Occurred())) {
FormatTypeError(arg, "int, float");
return false;
}
return true;
@ -664,8 +639,9 @@ bool CheckAndGetFloat(PyObject* arg, float* value) {
bool CheckAndGetBool(PyObject* arg, bool* value) {
long long_value = PyLong_AsLong(arg); // NOLINT
if (long_value == -1 && PyErr_Occurred()) {
FormatTypeError(arg, "int, long, bool");
if (!strcmp(Py_TYPE(arg)->tp_name, "numpy.ndarray") ||
(long_value == -1 && PyErr_Occurred())) {
FormatTypeError(arg, "int, bool");
return false;
}
*value = static_cast<bool>(long_value);

View File

@ -60,7 +60,7 @@ class ScopedPythonPtr {
// This function must be called with a reference that you own.
// this->reset(this->get()) is wrong!
// this->reset(this->release()) is OK.
PyObjectStruct* reset(PyObjectStruct* p = NULL) {
PyObjectStruct* reset(PyObjectStruct* p = nullptr) {
Py_XDECREF(ptr_);
ptr_ = p;
return ptr_;

View File

@ -197,7 +197,7 @@ bool TryCreateParentDirectory(const std::string& prefix,
bool GetProtocAbsolutePath(std::string* path) {
#ifdef _WIN32
char buffer[MAX_PATH];
int len = GetModuleFileNameA(NULL, buffer, MAX_PATH);
int len = GetModuleFileNameA(nullptr, buffer, MAX_PATH);
#elif defined(__APPLE__)
char buffer[PATH_MAX];
int len = 0;
@ -212,7 +212,7 @@ bool GetProtocAbsolutePath(std::string* path) {
char buffer[PATH_MAX];
size_t len = PATH_MAX;
int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1};
if (sysctl(mib, 4, &buffer, &len, NULL, 0) != 0) {
if (sysctl(mib, 4, &buffer, &len, nullptr, 0) != 0) {
len = 0;
}
#else
@ -2307,7 +2307,7 @@ bool CommandLineInterface::GeneratePluginOutput(
if (!output_file.insertion_point().empty()) {
std::string filename = output_file.name();
// Open a file for insert.
// We reset current_output to NULL first so that the old file is closed
// We reset current_output to nullptr first so that the old file is closed
// before the new one is opened.
current_output.reset();
current_output.reset(
@ -2316,7 +2316,7 @@ bool CommandLineInterface::GeneratePluginOutput(
output_file.generated_code_info()));
} else if (!output_file.name().empty()) {
// Starting a new file. Open it.
// We reset current_output to NULL first so that the old file is closed
// We reset current_output to nullptr first so that the old file is closed
// before the new one is opened.
current_output.reset();
current_output.reset(generator_context->Open(output_file.name()));

View File

@ -1714,7 +1714,7 @@ TEST_F(CommandLineInterfaceTest, WriteDependencyManifestFile) {
" optional Foo foo = 1;\n"
"}\n");
std::string current_working_directory = getcwd(NULL, 0);
std::string current_working_directory = getcwd(nullptr, 0);
SwitchToTempDirectory();
Run("protocol_compiler --dependency_out=manifest --test_out=. "

View File

@ -498,7 +498,7 @@ io::ZeroCopyInputStream* DiskSourceTree::OpenDiskFile(
#if defined(_WIN32)
if (ret == 0 && sb.st_mode & S_IFDIR) {
last_error_message_ = "Input file is a directory.";
return NULL;
return nullptr;
}
#else
if (ret == 0 && S_ISDIR(sb.st_mode)) {

View File

@ -221,12 +221,8 @@ bool Parser::Consume(const char* text, const char* error) {
}
bool Parser::Consume(const char* text) {
if (TryConsume(text)) {
return true;
} else {
AddError("Expected \"" + std::string(text) + "\".");
return false;
}
std::string error = "Expected \"" + std::string(text) + "\".";
return Consume(text, error.c_str());
}
bool Parser::ConsumeIdentifier(std::string* output, const char* error) {

View File

@ -55,6 +55,16 @@ template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedField<float>;
template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedField<double>;
template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedPtrField<std::string>;
namespace internal {
template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedIterator<bool>;
template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedIterator<int32_t>;
template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedIterator<uint32_t>;
template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedIterator<int64_t>;
template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedIterator<uint64_t>;
template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedIterator<float>;
template class PROTOBUF_EXPORT_TEMPLATE_DEFINE RepeatedIterator<double>;
} // namespace internal
} // namespace protobuf
} // namespace google

View File

@ -149,6 +149,9 @@ PROTO_MEMSWAP_DEF_SIZE(uint64_t, (1u << 31))
#undef PROTO_MEMSWAP_DEF_SIZE
template <typename Element>
class RepeatedIterator;
} // namespace internal
// RepeatedField is used to represent repeated fields of a primitive type (in
@ -257,8 +260,8 @@ class RepeatedField final {
void SwapElements(int index1, int index2);
// STL-like iterator support
typedef Element* iterator;
typedef const Element* const_iterator;
typedef internal::RepeatedIterator<Element> iterator;
typedef internal::RepeatedIterator<const Element> const_iterator;
typedef Element value_type;
typedef value_type& reference;
typedef const value_type& const_reference;
@ -856,31 +859,31 @@ void RepeatedField<Element>::SwapElements(int index1, int index2) {
template <typename Element>
inline typename RepeatedField<Element>::iterator
RepeatedField<Element>::begin() {
return unsafe_elements();
return iterator(unsafe_elements());
}
template <typename Element>
inline typename RepeatedField<Element>::const_iterator
RepeatedField<Element>::begin() const {
return unsafe_elements();
return const_iterator(unsafe_elements());
}
template <typename Element>
inline typename RepeatedField<Element>::const_iterator
RepeatedField<Element>::cbegin() const {
return unsafe_elements();
return const_iterator(unsafe_elements());
}
template <typename Element>
inline typename RepeatedField<Element>::iterator RepeatedField<Element>::end() {
return unsafe_elements() + current_size_;
return iterator(unsafe_elements() + current_size_);
}
template <typename Element>
inline typename RepeatedField<Element>::const_iterator
RepeatedField<Element>::end() const {
return unsafe_elements() + current_size_;
return const_iterator(unsafe_elements() + current_size_);
}
template <typename Element>
inline typename RepeatedField<Element>::const_iterator
RepeatedField<Element>::cend() const {
return unsafe_elements() + current_size_;
return const_iterator(unsafe_elements() + current_size_);
}
template <typename Element>
@ -1022,6 +1025,131 @@ struct ElementCopier<Element, true> {
// Ported by johannes from util/gtl/proto-array-iterators.h
namespace internal {
// STL-like iterator implementation for RepeatedField. You should not
// refer to this class directly; use RepeatedField<T>::iterator instead.
//
// Note: All of the iterator operators *must* be inlined to avoid performance
// regressions. This is caused by the extern template declarations below (which
// are required because of the RepeatedField extern template declarations). If
// any of these functions aren't explicitly inlined (e.g. defined in the class),
// the compiler isn't allowed to inline them.
template <typename Element>
class RepeatedIterator {
public:
using iterator_category = std::random_access_iterator_tag;
// Note: remove_const is necessary for std::partial_sum, which uses value_type
// to determine the summation variable type.
using value_type = typename std::remove_const<Element>::type;
using difference_type = std::ptrdiff_t;
using pointer = Element*;
using reference = Element&;
constexpr RepeatedIterator() noexcept : it_(nullptr) {}
// Allows "upcasting" from RepeatedIterator<T**> to
// RepeatedIterator<const T*const*>.
template <typename OtherElement,
typename std::enable_if<std::is_convertible<
OtherElement*, pointer>::value>::type* = nullptr>
constexpr RepeatedIterator(
const RepeatedIterator<OtherElement>& other) noexcept
: it_(other.it_) {}
// dereferenceable
constexpr reference operator*() const noexcept { return *it_; }
constexpr pointer operator->() const noexcept { return it_; }
private:
// Helper alias to hide the internal type.
using iterator = RepeatedIterator<Element>;
public:
// {inc,dec}rementable
iterator& operator++() noexcept {
++it_;
return *this;
}
iterator operator++(int) noexcept { return iterator(it_++); }
iterator& operator--() noexcept {
--it_;
return *this;
}
iterator operator--(int) noexcept { return iterator(it_--); }
// equality_comparable
friend constexpr bool operator==(const iterator& x,
const iterator& y) noexcept {
return x.it_ == y.it_;
}
friend constexpr bool operator!=(const iterator& x,
const iterator& y) noexcept {
return x.it_ != y.it_;
}
// less_than_comparable
friend constexpr bool operator<(const iterator& x,
const iterator& y) noexcept {
return x.it_ < y.it_;
}
friend constexpr bool operator<=(const iterator& x,
const iterator& y) noexcept {
return x.it_ <= y.it_;
}
friend constexpr bool operator>(const iterator& x,
const iterator& y) noexcept {
return x.it_ > y.it_;
}
friend constexpr bool operator>=(const iterator& x,
const iterator& y) noexcept {
return x.it_ >= y.it_;
}
// addable, subtractable
iterator& operator+=(difference_type d) noexcept {
it_ += d;
return *this;
}
constexpr iterator operator+(difference_type d) const noexcept {
return iterator(it_ + d);
}
friend constexpr iterator operator+(const difference_type d,
iterator it) noexcept {
return it + d;
}
iterator& operator-=(difference_type d) noexcept {
it_ -= d;
return *this;
}
iterator constexpr operator-(difference_type d) const noexcept {
return iterator(it_ - d);
}
// indexable
constexpr reference operator[](difference_type d) const noexcept {
return it_[d];
}
// random access iterator
friend constexpr difference_type operator-(iterator it1,
iterator it2) noexcept {
return it1.it_ - it2.it_;
}
private:
template <typename OtherElement>
friend class RepeatedIterator;
// Allow construction from RepeatedField.
friend class RepeatedField<Element>; // TODO(b/218695758) Remove this.
friend class RepeatedField<value_type>;
explicit RepeatedIterator(Element* it) noexcept : it_(it) {}
// The internal iterator.
Element* it_;
};
// A back inserter for RepeatedField objects.
template <typename T>
class RepeatedFieldBackInsertIterator {
@ -1068,6 +1196,20 @@ extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedField<uint64_t>;
extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedField<float>;
extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedField<double>;
namespace internal {
extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedIterator<bool>;
extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE
RepeatedIterator<int32_t>;
extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE
RepeatedIterator<uint32_t>;
extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE
RepeatedIterator<int64_t>;
extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE
RepeatedIterator<uint64_t>;
extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedIterator<float>;
extern template class PROTOBUF_EXPORT_TEMPLATE_DECLARE RepeatedIterator<double>;
} // namespace internal
} // namespace protobuf
} // namespace google

View File

@ -1739,8 +1739,8 @@ TEST_F(RepeatedFieldIteratorTest, ConstIteration) {
EXPECT_EQ(1, *iter++);
EXPECT_EQ(2, *iter);
++iter;
EXPECT_TRUE(proto_array_.end() == iter);
EXPECT_EQ(2, *(proto_array_.end() - 1));
EXPECT_TRUE(const_proto_array.end() == iter);
EXPECT_EQ(2, *(const_proto_array.end() - 1));
}
TEST_F(RepeatedFieldIteratorTest, Mutation) {
@ -1784,6 +1784,7 @@ TEST_F(RepeatedPtrFieldIteratorTest, ConstIteration) {
const RepeatedPtrField<std::string>& const_proto_array = proto_array_;
RepeatedPtrField<std::string>::const_iterator iter =
const_proto_array.begin();
iter - const_proto_array.cbegin();
EXPECT_EQ("foo", *iter);
++iter;
EXPECT_EQ("bar", *(iter++));

View File

@ -102,13 +102,18 @@ def _impl(ctx):
],
)
if 'osx' in ctx.attr.target_full_name:
sysroot_action_set = all_link_actions
else:
sysroot_action_set = all_link_actions + all_compile_actions
sysroot_flags = feature(
name = "sysroot_flags",
#Only enable this if a sysroot was specified
enabled = (ctx.attr.sysroot != ""),
flag_sets = [
flag_set(
actions = all_link_actions + all_compile_actions,
actions = sysroot_action_set,
flag_groups = [
flag_group(
flags = [