AuroraMiddleware/protobuf
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Adds the ability to ignore unknown fields on parse

Browse Source 
This adds the ability for the MRI Ruby library to optionally pass in a
ignore_unknown_fields option when decoding JSON. The functionality was
added upstream in upb, this change exposes that option.
This commit is contained in:
Erik Benoist 2018-11-17 16:50:24 -06:00
parent 436139803f
commit 9ac11326df
8 changed files with 4288 additions and 1875 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
conformance/failure_list_ruby.txt
View File

@ -5,8 +5,6 @@ Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
Recommended.Proto3.JsonInput.DurationHas3FractionalDigits.Validator
Recommended.Proto3.JsonInput.DurationHas6FractionalDigits.Validator
Recommended.Proto3.JsonInput.DurationHas9FractionalDigits.Validator
Recommended.Proto3.JsonInput.DurationHasZeroFractionalDigit.Validator
Recommended.Proto3.JsonInput.Int64FieldBeString.Validator
Recommended.Proto3.JsonInput.MapFieldValueIsNull
Recommended.Proto3.JsonInput.RepeatedFieldMessageElementIsNull
@ -17,9 +15,6 @@ Recommended.Proto3.JsonInput.StringFieldUnpairedHighSurrogate
Recommended.Proto3.JsonInput.StringFieldUnpairedLowSurrogate
Recommended.Proto3.JsonInput.TimestampHas3FractionalDigits.Validator
Recommended.Proto3.JsonInput.TimestampHas6FractionalDigits.Validator
Recommended.Proto3.JsonInput.TimestampHas9FractionalDigits.Validator
Recommended.Proto3.JsonInput.TimestampHasZeroFractionalDigit.Validator
Recommended.Proto3.JsonInput.TimestampZeroNormalized.Validator
Recommended.Proto3.JsonInput.Uint64FieldBeString.Validator
Required.DurationProtoInputTooLarge.JsonOutput
Required.DurationProtoInputTooSmall.JsonOutput
@ -48,12 +43,8 @@ Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.ProtobufOutput
Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.JsonOutput
Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.ProtobufOutput
Required.Proto3.JsonInput.DoubleFieldNan.JsonOutput
Required.Proto3.JsonInput.DurationMaxValue.JsonOutput
Required.Proto3.JsonInput.DurationMaxValue.ProtobufOutput
Required.Proto3.JsonInput.DurationMinValue.JsonOutput
Required.Proto3.JsonInput.DurationMinValue.ProtobufOutput
Required.Proto3.JsonInput.DurationRepeatedValue.JsonOutput
Required.Proto3.JsonInput.DurationRepeatedValue.ProtobufOutput
Required.Proto3.JsonInput.FieldMask.JsonOutput
Required.Proto3.JsonInput.FieldMask.ProtobufOutput
Required.Proto3.JsonInput.FloatFieldInfinity.JsonOutput
@ -61,71 +52,32 @@ Required.Proto3.JsonInput.FloatFieldNan.JsonOutput
Required.Proto3.JsonInput.FloatFieldNegativeInfinity.JsonOutput
Required.Proto3.JsonInput.OneofFieldDuplicate
Required.Proto3.JsonInput.OptionalBoolWrapper.JsonOutput
Required.Proto3.JsonInput.OptionalBoolWrapper.ProtobufOutput
Required.Proto3.JsonInput.OptionalBytesWrapper.JsonOutput
Required.Proto3.JsonInput.OptionalBytesWrapper.ProtobufOutput
Required.Proto3.JsonInput.OptionalDoubleWrapper.JsonOutput
Required.Proto3.JsonInput.OptionalDoubleWrapper.ProtobufOutput
Required.Proto3.JsonInput.OptionalFloatWrapper.JsonOutput
Required.Proto3.JsonInput.OptionalFloatWrapper.ProtobufOutput
Required.Proto3.JsonInput.OptionalInt32Wrapper.JsonOutput
Required.Proto3.JsonInput.OptionalInt32Wrapper.ProtobufOutput
Required.Proto3.JsonInput.OptionalInt64Wrapper.JsonOutput
Required.Proto3.JsonInput.OptionalInt64Wrapper.ProtobufOutput
Required.Proto3.JsonInput.OptionalStringWrapper.JsonOutput
Required.Proto3.JsonInput.OptionalStringWrapper.ProtobufOutput
Required.Proto3.JsonInput.OptionalUint32Wrapper.JsonOutput
Required.Proto3.JsonInput.OptionalUint32Wrapper.ProtobufOutput
Required.Proto3.JsonInput.OptionalUint64Wrapper.JsonOutput
Required.Proto3.JsonInput.OptionalUint64Wrapper.ProtobufOutput
Required.Proto3.JsonInput.OptionalWrapperTypesWithNonDefaultValue.JsonOutput
Required.Proto3.JsonInput.OptionalWrapperTypesWithNonDefaultValue.ProtobufOutput
Required.Proto3.JsonInput.RepeatedBoolWrapper.JsonOutput
Required.Proto3.JsonInput.RepeatedBoolWrapper.ProtobufOutput
Required.Proto3.JsonInput.RepeatedBytesWrapper.JsonOutput
Required.Proto3.JsonInput.RepeatedBytesWrapper.ProtobufOutput
Required.Proto3.JsonInput.RepeatedDoubleWrapper.JsonOutput
Required.Proto3.JsonInput.RepeatedDoubleWrapper.ProtobufOutput
Required.Proto3.JsonInput.RepeatedFloatWrapper.JsonOutput
Required.Proto3.JsonInput.RepeatedFloatWrapper.ProtobufOutput
Required.Proto3.JsonInput.RepeatedInt32Wrapper.JsonOutput
Required.Proto3.JsonInput.RepeatedInt32Wrapper.ProtobufOutput
Required.Proto3.JsonInput.RepeatedInt64Wrapper.JsonOutput
Required.Proto3.JsonInput.RepeatedInt64Wrapper.ProtobufOutput
Required.Proto3.JsonInput.RepeatedStringWrapper.JsonOutput
Required.Proto3.JsonInput.RepeatedStringWrapper.ProtobufOutput
Required.Proto3.JsonInput.RepeatedUint32Wrapper.JsonOutput
Required.Proto3.JsonInput.RepeatedUint32Wrapper.ProtobufOutput
Required.Proto3.JsonInput.RepeatedUint64Wrapper.JsonOutput
Required.Proto3.JsonInput.RepeatedUint64Wrapper.ProtobufOutput
Required.Proto3.JsonInput.StringFieldSurrogatePair.JsonOutput
Required.Proto3.JsonInput.StringFieldSurrogatePair.ProtobufOutput
Required.Proto3.JsonInput.Struct.JsonOutput
Required.Proto3.JsonInput.Struct.ProtobufOutput
Required.Proto3.JsonInput.TimestampMaxValue.JsonOutput
Required.Proto3.JsonInput.TimestampMaxValue.ProtobufOutput
Required.Proto3.JsonInput.TimestampJsonInputTooSmall
Required.Proto3.JsonInput.TimestampMinValue.JsonOutput
Required.Proto3.JsonInput.TimestampMinValue.ProtobufOutput
Required.Proto3.JsonInput.TimestampRepeatedValue.JsonOutput
Required.Proto3.JsonInput.TimestampRepeatedValue.ProtobufOutput
Required.Proto3.JsonInput.TimestampWithNegativeOffset.JsonOutput
Required.Proto3.JsonInput.TimestampWithNegativeOffset.ProtobufOutput
Required.Proto3.JsonInput.TimestampWithPositiveOffset.JsonOutput
Required.Proto3.JsonInput.TimestampWithPositiveOffset.ProtobufOutput
Required.Proto3.JsonInput.ValueAcceptBool.JsonOutput
Required.Proto3.JsonInput.ValueAcceptBool.ProtobufOutput
Required.Proto3.JsonInput.ValueAcceptFloat.JsonOutput
Required.Proto3.JsonInput.ValueAcceptFloat.ProtobufOutput
Required.Proto3.JsonInput.ValueAcceptInteger.JsonOutput
Required.Proto3.JsonInput.ValueAcceptInteger.ProtobufOutput
Required.Proto3.JsonInput.ValueAcceptList.JsonOutput
Required.Proto3.JsonInput.ValueAcceptList.ProtobufOutput
Required.Proto3.JsonInput.ValueAcceptNull.JsonOutput
Required.Proto3.JsonInput.ValueAcceptNull.ProtobufOutput
Required.Proto3.JsonInput.ValueAcceptObject.JsonOutput
Required.Proto3.JsonInput.ValueAcceptObject.ProtobufOutput
Required.Proto3.JsonInput.ValueAcceptString.JsonOutput
Required.Proto3.JsonInput.ValueAcceptString.ProtobufOutput
Required.Proto3.ProtobufInput.DoubleFieldNormalizeQuietNan.JsonOutput
Required.Proto3.ProtobufInput.DoubleFieldNormalizeSignalingNan.JsonOutput
Required.Proto3.ProtobufInput.FloatFieldNormalizeQuietNan.JsonOutput

30
ruby/ext/google/protobuf_c/encode_decode.c
View File

@ -891,19 +891,38 @@ VALUE Message_decode(VALUE klass, VALUE data) {
/*
* call-seq:
* MessageClass.decode_json(data) => message
* MessageClass.decode_json(data, options = {}) => message
*
* Decodes the given data (as a string containing bytes in protocol buffers wire
* format) under the interpretration given by this message class's definition
* and returns a message object with the corresponding field values.
*
* @param options [Hash] options for the decoder
* ignore_unknown_fields: set true to ignore unknown fields (default is to raise an error)
*/
VALUE Message_decode_json(VALUE klass, VALUE data) {
VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass) {
VALUE descriptor = rb_ivar_get(klass, descriptor_instancevar_interned);
Descriptor* desc = ruby_to_Descriptor(descriptor);
VALUE msgklass = Descriptor_msgclass(descriptor);
VALUE msg_rb;
VALUE data = argv[0];
VALUE ignore_unknown_fields = Qfalse;
MessageHeader* msg;
if (argc < 1 || argc > 2) {
rb_raise(rb_eArgError, "Expected 1 or 2 arguments.");
}
if (argc == 2) {
VALUE hash_args = argv[1];
if (TYPE(hash_args) != T_HASH) {
rb_raise(rb_eArgError, "Expected hash arguments.");
}
ignore_unknown_fields = rb_hash_lookup2(
hash_args, ID2SYM(rb_intern("ignore_unknown_fields")), Qfalse);
}
if (TYPE(data) != T_STRING) {
rb_raise(rb_eArgError, "Expected string for JSON data.");
}
@ -922,7 +941,7 @@ VALUE Message_decode_json(VALUE klass, VALUE data) {
stackenv_init(&se, "Error occurred during parsing: %s");
upb_sink_reset(&sink, get_fill_handlers(desc), msg);
parser = upb_json_parser_create(&se.env, method, &sink);
parser = upb_json_parser_create(&se.env, method, &sink, ignore_unknown_fields);
upb_bufsrc_putbuf(RSTRING_PTR(data), RSTRING_LEN(data),
upb_json_parser_input(parser));
@ -1310,9 +1329,12 @@ VALUE Message_encode(VALUE klass, VALUE msg_rb) {
/*
* call-seq:
* MessageClass.encode_json(msg) => json_string
* MessageClass.encode_json(msg, options = {}) => json_string
*
* Encodes the given message object into its serialized JSON representation.
* @param options [Hash] options for the decoder
* preserve_proto_fieldnames: set true to use original fieldnames (default is to camelCase)
* emit_defaults: set true to emit 0/false values (default is to omit them)
*/
VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass) {
VALUE descriptor = rb_ivar_get(klass, descriptor_instancevar_interned);

2
ruby/ext/google/protobuf_c/message.c
View File

@ -628,7 +628,7 @@ VALUE build_class_from_descriptor(Descriptor* desc) {
rb_define_method(klass, "[]=", Message_index_set, 2);
rb_define_singleton_method(klass, "decode", Message_decode, 1);
rb_define_singleton_method(klass, "encode", Message_encode, 1);
rb_define_singleton_method(klass, "decode_json", Message_decode_json, 1);
rb_define_singleton_method(klass, "decode_json", Message_decode_json, -1);
rb_define_singleton_method(klass, "encode_json", Message_encode_json, -1);
rb_define_singleton_method(klass, "descriptor", Message_descriptor, 0);

2
ruby/ext/google/protobuf_c/protobuf.h
View File

@ -570,7 +570,7 @@ VALUE Message_index_set(VALUE _self, VALUE field_name, VALUE value);
VALUE Message_descriptor(VALUE klass);
VALUE Message_decode(VALUE klass, VALUE data);
VALUE Message_encode(VALUE klass, VALUE msg_rb);
VALUE Message_decode_json(VALUE klass, VALUE data);
VALUE Message_decode_json(int argc, VALUE* argv, VALUE klass);
VALUE Message_encode_json(int argc, VALUE* argv, VALUE klass);
VALUE Google_Protobuf_discard_unknown(VALUE self, VALUE msg_rb);

4586
ruby/ext/google/protobuf_c/upb.c
View File

File diff suppressed because it is too large Load Diff

1476
ruby/ext/google/protobuf_c/upb.h
View File

File diff suppressed because it is too large Load Diff

4
ruby/lib/google/protobuf.rb
View File

@ -69,8 +69,8 @@ module Google
klass.decode(proto)
end
def self.decode_json(klass, json)
klass.decode_json(json)
def self.decode_json(klass, json, options = {})
klass.decode_json(json, options)
end
end

13
ruby/tests/basic.rb
View File

@ -254,6 +254,19 @@ module BasicTest
"b" => TestMessage2.new(:foo => 2)}
end
def test_protobuf_decode_json_ignore_unknown_fields
m = TestMessage.decode_json({
optional_string: "foo",
not_in_message: "some_value"
}.to_json, { ignore_unknown_fields: true })
assert_equal m.optional_string, "foo"
e = assert_raise Google::Protobuf::ParseError do
TestMessage.decode_json({ not_in_message: "some_value" }.to_json)
end
assert_match(/No such field: not_in_message/, e.message)
end
def test_to_h
m = TestMessage.new(:optional_bool => true, :optional_double => -10.100001, :optional_string => 'foo', :repeated_string => ['bar1', 'bar2'], :repeated_msg => [TestMessage2.new(:foo => 100)])
expected_result = {