Merge branch 'master' of github.com:google/protobuf

This commit is contained in:
Jisi Liu 2016-04-01 16:46:49 -07:00
commit cba75ad404
12 changed files with 218 additions and 106 deletions

View File

@ -0,0 +1,37 @@
<?xml version="1.0" encoding="utf-8"?>
<package>
<metadata>
<id>Google.Protobuf.Tools</id>
<title>Google Protocol Buffers tools</title>
<summary>Tools for Protocol Buffers - Google's data interchange format.</summary>
<description>See project site for more info.</description>
<version>3.0.0-beta2</version>
<authors>Google Inc.</authors>
<owners>protobuf-packages</owners>
<licenseUrl>https://github.com/google/protobuf/blob/master/LICENSE</licenseUrl>
<projectUrl>https://github.com/google/protobuf</projectUrl>
<requireLicenseAcceptance>false</requireLicenseAcceptance>
<releaseNotes>Tools for Protocol Buffers</releaseNotes>
<copyright>Copyright 2015, Google Inc.</copyright>
<tags>Protocol Buffers Binary Serialization Format Google proto proto3</tags>
</metadata>
<files>
<file src="protoc\windows_x86\protoc.exe" target="tools\windows_x86\protoc.exe" />
<file src="protoc\windows_x64\protoc.exe" target="tools\windows_x64\protoc.exe" />
<file src="protoc\linux_x86\protoc" target="tools\linux_x86\protoc" />
<file src="protoc\linux_x64\protoc" target="tools\linux_x64\protoc" />
<file src="protoc\macosx_x86\protoc" target="tools\macosx_x86\protoc" />
<file src="protoc\macosx_x64\protoc" target="tools\macosx_x64\protoc" />
<file src="..\src\google\protobuf\any.proto" target="tools\google\protobuf" />
<file src="..\src\google\protobuf\api.proto" target="tools\google\protobuf" />
<file src="..\src\google\protobuf\descriptor.proto" target="tools\google\protobuf" />
<file src="..\src\google\protobuf\duration.proto" target="tools\google\protobuf" />
<file src="..\src\google\protobuf\empty.proto" target="tools\google\protobuf" />
<file src="..\src\google\protobuf\field_mask.proto" target="tools\google\protobuf" />
<file src="..\src\google\protobuf\source_context.proto" target="tools\google\protobuf" />
<file src="..\src\google\protobuf\struct.proto" target="tools\google\protobuf" />
<file src="..\src\google\protobuf\timestamp.proto" target="tools\google\protobuf" />
<file src="..\src\google\protobuf\type.proto" target="tools\google\protobuf" />
<file src="..\src\google\protobuf\wrappers.proto" target="tools\google\protobuf" />
</files>
</package>

View File

@ -321,7 +321,6 @@ namespace Google.Protobuf
[TestCase("1970-01-01T00:00:00.001Z", 1000000)]
[TestCase("1970-01-01T00:00:00.010Z", 10000000)]
[TestCase("1970-01-01T00:00:00.100Z", 100000000)]
[TestCase("1970-01-01T00:00:00.100Z", 100000000)]
[TestCase("1970-01-01T00:00:00.120Z", 120000000)]
[TestCase("1970-01-01T00:00:00.123Z", 123000000)]
[TestCase("1970-01-01T00:00:00.123400Z", 123400000)]

View File

@ -249,7 +249,6 @@ namespace Google.Protobuf
[TestCase("[,", 1)]
[TestCase("{", 1)]
[TestCase("{,", 1)]
[TestCase("{", 1)]
[TestCase("{[", 1)]
[TestCase("{{", 1)]
[TestCase("{0", 1)]

View File

@ -23,7 +23,6 @@
<group targetFramework="xamarin.ios" />
<group targetFramework="monotouch" />
<group targetFramework="monoandroid" />
<!-- Dependencies for newer, more granular platforms (.NET Core etc) -->
<group targetFramework="dotnet">
<dependency id="System.Collections" version="4.0.0" />
@ -49,18 +48,5 @@
<file src="bin/ReleaseSigned/Google.Protobuf.pdb" target="lib/dotnet" />
<file src="bin/ReleaseSigned/Google.Protobuf.xml" target="lib/dotnet" />
<file src="**\*.cs" target="src" />
<file src="..\..\..\cmake\Release\protoc.exe" target="tools" />
<file src="..\..\..\src\google\protobuf\any.proto" target="tools\google\protobuf" />
<file src="..\..\..\src\google\protobuf\api.proto" target="tools\google\protobuf" />
<file src="..\..\..\src\google\protobuf\descriptor.proto" target="tools\google\protobuf" />
<file src="..\..\..\src\google\protobuf\duration.proto" target="tools\google\protobuf" />
<file src="..\..\..\src\google\protobuf\empty.proto" target="tools\google\protobuf" />
<file src="..\..\..\src\google\protobuf\field_mask.proto" target="tools\google\protobuf" />
<file src="..\..\..\src\google\protobuf\source_context.proto" target="tools\google\protobuf" />
<file src="..\..\..\src\google\protobuf\struct.proto" target="tools\google\protobuf" />
<file src="..\..\..\src\google\protobuf\timestamp.proto" target="tools\google\protobuf" />
<file src="..\..\..\src\google\protobuf\any.proto" target="tools\google\protobuf" />
<file src="..\..\..\src\google\protobuf\type.proto" target="tools\google\protobuf" />
<file src="..\..\..\src\google\protobuf\wrappers.proto" target="tools\google\protobuf" />
</files>
</package>

View File

@ -30,9 +30,8 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endregion
using System;
using System.Linq;
using Google.Protobuf.Compatibility;
using System;
namespace Google.Protobuf.Reflection
{
@ -41,20 +40,35 @@ namespace Google.Protobuf.Reflection
/// </summary>
public sealed class FieldDescriptor : DescriptorBase, IComparable<FieldDescriptor>
{
private readonly FieldDescriptorProto proto;
private EnumDescriptor enumType;
private MessageDescriptor messageType;
private readonly MessageDescriptor containingType;
private readonly OneofDescriptor containingOneof;
private FieldType fieldType;
private readonly string propertyName; // Annoyingly, needed in Crosslink.
private IFieldAccessor accessor;
/// <summary>
/// Get the field's containing message type.
/// </summary>
public MessageDescriptor ContainingType { get; }
/// <summary>
/// Returns the oneof containing this field, or <c>null</c> if it is not part of a oneof.
/// </summary>
public OneofDescriptor ContainingOneof { get; }
/// <summary>
/// The effective JSON name for this field. This is usually the lower-camel-cased form of the field name,
/// but can be overridden using the <c>json_name</c> option in the .proto file.
/// </summary>
public string JsonName { get; }
internal FieldDescriptorProto Proto { get; }
internal FieldDescriptor(FieldDescriptorProto proto, FileDescriptor file,
MessageDescriptor parent, int index, string propertyName)
: base(file, file.ComputeFullName(parent, proto.Name), index)
{
this.proto = proto;
Proto = proto;
if (proto.Type != 0)
{
fieldType = GetFieldTypeFromProtoType(proto.Type);
@ -64,7 +78,7 @@ namespace Google.Protobuf.Reflection
{
throw new DescriptorValidationException(this, "Field numbers must be positive integers.");
}
containingType = parent;
ContainingType = parent;
// OneofIndex "defaults" to -1 due to a hack in FieldDescriptor.OnConstruction.
if (proto.OneofIndex != -1)
{
@ -73,7 +87,7 @@ namespace Google.Protobuf.Reflection
throw new DescriptorValidationException(this,
$"FieldDescriptorProto.oneof_index is out of range for type {parent.Name}");
}
containingOneof = parent.Oneofs[proto.OneofIndex];
ContainingOneof = parent.Oneofs[proto.OneofIndex];
}
file.DescriptorPool.AddSymbol(this);
@ -83,20 +97,14 @@ namespace Google.Protobuf.Reflection
// We could trust the generated code and check whether the type of the property is
// a MapField, but that feels a tad nasty.
this.propertyName = propertyName;
JsonName = Proto.JsonName == "" ? JsonFormatter.ToCamelCase(Proto.Name) : Proto.JsonName;
}
/// <summary>
/// The brief name of the descriptor's target.
/// </summary>
public override string Name { get { return proto.Name; } }
/// <summary>
/// The json_name option of the descriptor's target.
/// </summary>
public string JsonName { get { return proto.JsonName == "" ? JsonFormatter.ToCamelCase(proto.Name) : proto.JsonName; } }
internal FieldDescriptorProto Proto { get { return proto; } }
public override string Name => Proto.Name;
/// <summary>
/// Returns the accessor for this field.
@ -116,7 +124,7 @@ namespace Google.Protobuf.Reflection
/// and this property will return null.
/// </para>
/// </remarks>
public IFieldAccessor Accessor { get { return accessor; } }
public IFieldAccessor Accessor => accessor;
/// <summary>
/// Maps a field type as included in the .proto file to a FieldType.
@ -169,62 +177,32 @@ namespace Google.Protobuf.Reflection
/// <summary>
/// Returns <c>true</c> if this field is a repeated field; <c>false</c> otherwise.
/// </summary>
public bool IsRepeated
{
get { return Proto.Label == FieldDescriptorProto.Types.Label.LABEL_REPEATED; }
}
public bool IsRepeated => Proto.Label == FieldDescriptorProto.Types.Label.LABEL_REPEATED;
/// <summary>
/// Returns <c>true</c> if this field is a map field; <c>false</c> otherwise.
/// </summary>
public bool IsMap
{
get { return fieldType == FieldType.Message && messageType.Proto.Options != null && messageType.Proto.Options.MapEntry; }
}
public bool IsMap => fieldType == FieldType.Message && messageType.Proto.Options != null && messageType.Proto.Options.MapEntry;
/// <summary>
/// Returns <c>true</c> if this field is a packed, repeated field; <c>false</c> otherwise.
/// </summary>
public bool IsPacked
{
public bool IsPacked =>
// Note the || rather than && here - we're effectively defaulting to packed, because that *is*
// the default in proto3, which is all we support. We may give the wrong result for the protos
// within descriptor.proto, but that's okay, as they're never exposed and we don't use IsPacked
// within the runtime.
get { return Proto.Options == null || Proto.Options.Packed; }
}
/// <summary>
/// Get the field's containing message type.
/// </summary>
public MessageDescriptor ContainingType
{
get { return containingType; }
}
/// <summary>
/// Returns the oneof containing this field, or <c>null</c> if it is not part of a oneof.
/// </summary>
public OneofDescriptor ContainingOneof
{
get { return containingOneof; }
}
Proto.Options == null || Proto.Options.Packed;
/// <summary>
/// Returns the type of the field.
/// </summary>
public FieldType FieldType
{
get { return fieldType; }
}
public FieldType FieldType => fieldType;
/// <summary>
/// Returns the field number declared in the proto file.
/// </summary>
public int FieldNumber
{
get { return Proto.Number; }
}
public int FieldNumber => Proto.Number;
/// <summary>
/// Compares this descriptor with another one, ordering in "canonical" order
@ -234,7 +212,7 @@ namespace Google.Protobuf.Reflection
/// </summary>
public int CompareTo(FieldDescriptor other)
{
if (other.containingType != containingType)
if (other.ContainingType != ContainingType)
{
throw new ArgumentException("FieldDescriptors can only be compared to other FieldDescriptors " +
"for fields of the same message type.");
@ -337,14 +315,14 @@ namespace Google.Protobuf.Reflection
File.DescriptorPool.AddFieldByNumber(this);
if (containingType != null && containingType.Proto.Options != null && containingType.Proto.Options.MessageSetWireFormat)
if (ContainingType != null && ContainingType.Proto.Options != null && ContainingType.Proto.Options.MessageSetWireFormat)
{
throw new DescriptorValidationException(this, "MessageSet format is not supported.");
}
accessor = CreateAccessor(propertyName);
accessor = CreateAccessor();
}
private IFieldAccessor CreateAccessor(string propertyName)
private IFieldAccessor CreateAccessor()
{
// If we're given no property name, that's because we really don't want an accessor.
// (At the moment, that means it's a map entry message...)
@ -352,10 +330,10 @@ namespace Google.Protobuf.Reflection
{
return null;
}
var property = containingType.ClrType.GetProperty(propertyName);
var property = ContainingType.ClrType.GetProperty(propertyName);
if (property == null)
{
throw new DescriptorValidationException(this, $"Property {propertyName} not found in {containingType.ClrType}");
throw new DescriptorValidationException(this, $"Property {propertyName} not found in {ContainingType.ClrType}");
}
return IsMap ? new MapFieldAccessor(property, this)
: IsRepeated ? new RepeatedFieldAccessor(property, this)

View File

@ -109,6 +109,15 @@ NS_ASSUME_NONNULL_BEGIN
/// or until an endgroup tag, whichever comes first.
- (void)skipMessage;
/// Check to see if the logical end of the stream has been reached.
///
/// This can return NO when there is no more data, but the current parsing
/// expected more data.
- (BOOL)isAtEnd;
/// The offset into the stream.
- (size_t)position;
/// Verifies that the last call to @c -readTag returned the given tag value.
/// This is used to verify that a nested group ended with the correct end tag.
/// Throws @c NSParseErrorException if value does not match the last tag.

View File

@ -359,6 +359,14 @@ void GPBCodedInputStreamCheckLastTagWas(GPBCodedInputStreamState *state,
}
}
- (BOOL)isAtEnd {
return GPBCodedInputStreamIsAtEnd(&state_);
}
- (size_t)position {
return state_.bufferPos;
}
- (double)readDouble {
return GPBCodedInputStreamReadDouble(&state_);
}

View File

@ -123,13 +123,5 @@ C++ Implementation
The C++ implementation for Python messages is built as a Python extension to
improve the overall protobuf Python performance.
To use the C++ implementation, you need to:
1) Install the C++ protobuf runtime library, please see instructions in the
parent directory.
2) Export an environment variable:
$ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
You must set this variable at runtime, before running your program, otherwise
the pure-Python implementation will be used. In a future release, we will
change the default so that C++ implementation is used whenever it is available.
To use the C++ implementation, you need to install the C++ protobuf runtime
library, please see instructions in the parent directory.

View File

@ -58,7 +58,11 @@ except ImportError:
import unittest
from google.protobuf.internal import _parameterized
from google.protobuf import descriptor_pb2
from google.protobuf import descriptor_pool
from google.protobuf import map_unittest_pb2
from google.protobuf import message_factory
from google.protobuf import text_format
from google.protobuf import unittest_pb2
from google.protobuf import unittest_proto3_arena_pb2
from google.protobuf.internal import api_implementation
@ -1763,5 +1767,60 @@ class PackedFieldTest(unittest.TestCase):
b'\x70\x01')
self.assertEqual(golden_data, message.SerializeToString())
@unittest.skipIf(api_implementation.Type() != 'cpp',
'explicit tests of the C++ implementation')
class OversizeProtosTest(unittest.TestCase):
def setUp(self):
self.file_desc = """
name: "f/f.msg2"
package: "f"
message_type {
name: "msg1"
field {
name: "payload"
number: 1
label: LABEL_OPTIONAL
type: TYPE_STRING
}
}
message_type {
name: "msg2"
field {
name: "field"
number: 1
label: LABEL_OPTIONAL
type: TYPE_MESSAGE
type_name: "msg1"
}
}
"""
pool = descriptor_pool.DescriptorPool()
desc = descriptor_pb2.FileDescriptorProto()
text_format.Parse(self.file_desc, desc)
pool.Add(desc)
self.proto_cls = message_factory.MessageFactory(pool).GetPrototype(
pool.FindMessageTypeByName('f.msg2'))
self.p = self.proto_cls()
self.p.field.payload = 'c' * (1024 * 1024 * 64 + 1)
self.p_serialized = self.p.SerializeToString()
def testAssertOversizeProto(self):
from google.protobuf.pyext._message import SetAllowOversizeProtos
SetAllowOversizeProtos(False)
q = self.proto_cls()
try:
q.ParseFromString(self.p_serialized)
except message.DecodeError as e:
self.assertEqual(str(e), 'Error parsing message')
def testSucceedOversizeProto(self):
from google.protobuf.pyext._message import SetAllowOversizeProtos
SetAllowOversizeProtos(True)
q = self.proto_cls()
q.ParseFromString(self.p_serialized)
self.assertEqual(self.p.field.payload, q.field.payload)
if __name__ == '__main__':
unittest.main()

View File

@ -1919,6 +1919,30 @@ static PyObject* CopyFrom(CMessage* self, PyObject* arg) {
Py_RETURN_NONE;
}
// Protobuf has a 64MB limit built in, this variable will override this. Please
// do not enable this unless you fully understand the implications: protobufs
// must all be kept in memory at the same time, so if they grow too big you may
// get OOM errors. The protobuf APIs do not provide any tools for processing
// protobufs in chunks. If you have protos this big you should break them up if
// it is at all convenient to do so.
static bool allow_oversize_protos = false;
// Provide a method in the module to set allow_oversize_protos to a boolean
// value. This method returns the newly value of allow_oversize_protos.
static PyObject* SetAllowOversizeProtos(PyObject* m, PyObject* arg) {
if (!arg || !PyBool_Check(arg)) {
PyErr_SetString(PyExc_TypeError,
"Argument to SetAllowOversizeProtos must be boolean");
return NULL;
}
allow_oversize_protos = PyObject_IsTrue(arg);
if (allow_oversize_protos) {
Py_RETURN_TRUE;
} else {
Py_RETURN_FALSE;
}
}
static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
const void* data;
Py_ssize_t data_length;
@ -1929,15 +1953,9 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
AssureWritable(self);
io::CodedInputStream input(
reinterpret_cast<const uint8*>(data), data_length);
#if PROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS
// Protobuf has a 64MB limit built in, this code will override this. Please do
// not enable this unless you fully understand the implications: protobufs
// must all be kept in memory at the same time, so if they grow too big you
// may get OOM errors. The protobuf APIs do not provide any tools for
// processing protobufs in chunks. If you have protos this big you should
// break them up if it is at all convenient to do so.
input.SetTotalBytesLimit(INT_MAX, INT_MAX);
#endif // PROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS
if (allow_oversize_protos) {
input.SetTotalBytesLimit(INT_MAX, INT_MAX);
}
PyDescriptorPool* pool = GetDescriptorPoolForMessage(self);
input.SetExtensionRegistry(pool->pool, pool->message_factory);
bool success = self->message->MergePartialFromCodedStream(&input);
@ -3058,6 +3076,11 @@ bool InitProto2MessageModule(PyObject *m) {
} // namespace python
} // namespace protobuf
static PyMethodDef ModuleMethods[] = {
{"SetAllowOversizeProtos",
(PyCFunction)google::protobuf::python::cmessage::SetAllowOversizeProtos,
METH_O, "Enable/disable oversize proto parsing."},
};
#if PY_MAJOR_VERSION >= 3
static struct PyModuleDef _module = {
@ -3065,7 +3088,7 @@ static struct PyModuleDef _module = {
"_message",
google::protobuf::python::module_docstring,
-1,
NULL,
ModuleMethods, /* m_methods */
NULL,
NULL,
NULL,
@ -3084,7 +3107,8 @@ extern "C" {
#if PY_MAJOR_VERSION >= 3
m = PyModule_Create(&_module);
#else
m = Py_InitModule3("_message", NULL, google::protobuf::python::module_docstring);
m = Py_InitModule3("_message", ModuleMethods,
google::protobuf::python::module_docstring);
#endif
if (m == NULL) {
return INITFUNC_ERRORVAL;

View File

@ -157,15 +157,30 @@ class test_conformance(_build_py):
status = subprocess.check_call(cmd, shell=True)
def get_option_from_sys_argv(option_str):
if option_str in sys.argv:
sys.argv.remove(option_str)
return True
return False
if __name__ == '__main__':
ext_module_list = []
cpp_impl = '--cpp_implementation'
warnings_as_errors = '--warnings_as_errors'
if cpp_impl in sys.argv:
sys.argv.remove(cpp_impl)
if get_option_from_sys_argv('--cpp_implementation'):
# Link libprotobuf.a and libprotobuf-lite.a statically with the
# extension. Note that those libraries have to be compiled with
# -fPIC for this to work.
compile_static_ext = get_option_from_sys_argv('--compile_static_extension')
extra_compile_args = ['-Wno-write-strings',
'-Wno-invalid-offsetof',
'-Wno-sign-compare']
libraries = ['protobuf']
extra_objects = None
if compile_static_ext:
libraries = None
extra_objects = ['../src/.libs/libprotobuf.a',
'../src/.libs/libprotobuf-lite.a']
test_conformance.target = 'test_python_cpp'
if "clang" in os.popen('$CC --version 2> /dev/null').read():
@ -176,16 +191,22 @@ if __name__ == '__main__':
sys.argv.remove(warnings_as_errors)
# C++ implementation extension
ext_module_list.append(
ext_module_list.extend([
Extension(
"google.protobuf.pyext._message",
glob.glob('google/protobuf/pyext/*.cc'),
include_dirs=[".", "../src"],
libraries=['protobuf'],
libraries=libraries,
extra_objects=extra_objects,
library_dirs=['../src/.libs'],
extra_compile_args=extra_compile_args,
)
)
),
Extension(
"google.protobuf.internal._api_implementation",
glob.glob('google/protobuf/internal/api_implementation.cc'),
extra_compile_args=['-DPYTHON_PROTO2_CPP_IMPL_V2'],
),
])
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
# Keep this list of dependencies in sync with tox.ini.

View File

@ -31,7 +31,7 @@
#include <google/protobuf/stubs/int128.h>
#include <iomanip>
#include <iostream> // NOLINT(readability/streams)
#include <ostream> // NOLINT(readability/streams)
#include <sstream>
namespace google {