AuroraMiddleware/protobuf
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Integrate from internal code base.

Browse Source
This commit is contained in:
Feng Xiao 2016-07-13 13:47:51 -07:00
parent 70c1ac756d
commit 9086d96439
59 changed files with 1139 additions and 464 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
cmake/examples.cmake
View File

@ -1,57 +1,57 @@
if(protobuf_VERBOSE)
message(STATUS "Protocol Buffers Examples Configuring...")
endif()
get_filename_component(examples_dir "../examples" ABSOLUTE)
if(protobuf_VERBOSE)
message(STATUS "Protocol Buffers Examples Configuring done")
endif()
include(ExternalProject)
# Internal utility function: Create a custom target representing a build of examples with custom options.
function(add_examples_build NAME)
ExternalProject_Add(${NAME}
PREFIX ${NAME}
SOURCE_DIR "${examples_dir}"
BINARY_DIR ${NAME}
STAMP_DIR ${NAME}/logs
INSTALL_COMMAND "" #Skip
LOG_CONFIGURE 1
CMAKE_CACHE_ARGS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}"
"-Dprotobuf_VERBOSE:BOOL=${protobuf_VERBOSE}"
${ARGN}
)
set_property(TARGET ${NAME} PROPERTY FOLDER "Examples")
set_property(TARGET ${NAME} PROPERTY EXCLUDE_FROM_ALL TRUE)
endfunction()
# Add examples as an external project.
# sub_directory cannot be used because the find_package(protobuf) call would cause failures with redefined targets.
add_examples_build(examples "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}")
add_dependencies(examples libprotobuf protoc)
option(protobuf_BUILD_EXAMPLES_MULTITEST "Build Examples in multiple configurations. Useful for testing." OFF)
mark_as_advanced(protobuf_BUILD_EXAMPLES_MULTITEST)
if(protobuf_BUILD_EXAMPLES_MULTITEST)
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
#Build using the legacy compatibility module.
add_examples_build(examples-legacy
"-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}"
"-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE"
)
add_dependencies(examples-legacy libprotobuf protoc)
#Build using the installed library.
add_examples_build(examples-installed
"-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}"
)
#Build using the installed library in legacy compatibility mode.
add_examples_build(examples-installed-legacy
"-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}"
"-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE"
)
endif()
if(protobuf_VERBOSE)
message(STATUS "Protocol Buffers Examples Configuring...")
endif()
get_filename_component(examples_dir "../examples" ABSOLUTE)
if(protobuf_VERBOSE)
message(STATUS "Protocol Buffers Examples Configuring done")
endif()
include(ExternalProject)
# Internal utility function: Create a custom target representing a build of examples with custom options.
function(add_examples_build NAME)
ExternalProject_Add(${NAME}
PREFIX ${NAME}
SOURCE_DIR "${examples_dir}"
BINARY_DIR ${NAME}
STAMP_DIR ${NAME}/logs
INSTALL_COMMAND "" #Skip
LOG_CONFIGURE 1
CMAKE_CACHE_ARGS "-DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}"
"-Dprotobuf_VERBOSE:BOOL=${protobuf_VERBOSE}"
${ARGN}
)
set_property(TARGET ${NAME} PROPERTY FOLDER "Examples")
set_property(TARGET ${NAME} PROPERTY EXCLUDE_FROM_ALL TRUE)
endfunction()
# Add examples as an external project.
# sub_directory cannot be used because the find_package(protobuf) call would cause failures with redefined targets.
add_examples_build(examples "-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}")
add_dependencies(examples libprotobuf protoc)
option(protobuf_BUILD_EXAMPLES_MULTITEST "Build Examples in multiple configurations. Useful for testing." OFF)
mark_as_advanced(protobuf_BUILD_EXAMPLES_MULTITEST)
if(protobuf_BUILD_EXAMPLES_MULTITEST)
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
#Build using the legacy compatibility module.
add_examples_build(examples-legacy
"-Dprotobuf_DIR:PATH=${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_CMAKEDIR}"
"-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE"
)
add_dependencies(examples-legacy libprotobuf protoc)
#Build using the installed library.
add_examples_build(examples-installed
"-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}"
)
#Build using the installed library in legacy compatibility mode.
add_examples_build(examples-installed-legacy
"-Dprotobuf_DIR:PATH=${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_CMAKEDIR}"
"-Dprotobuf_MODULE_COMPATIBLE:BOOL=TRUE"
)
endif()

14
cmake/protobuf-options.cmake
View File

@ -1,7 +1,7 @@
# Verbose output
option(protobuf_VERBOSE "Enable for verbose output" OFF)
mark_as_advanced(protobuf_VERBOSE)
# FindProtobuf module compatibel
option(protobuf_MODULE_COMPATIBLE "CMake build-in FindProtobuf.cmake module compatible" OFF)
mark_as_advanced(protobuf_MODULE_COMPATIBLE)
# Verbose output
option(protobuf_VERBOSE "Enable for verbose output" OFF)
mark_as_advanced(protobuf_VERBOSE)
# FindProtobuf module compatibel
option(protobuf_MODULE_COMPATIBLE "CMake build-in FindProtobuf.cmake module compatible" OFF)
mark_as_advanced(protobuf_MODULE_COMPATIBLE)

126
examples/CMakeLists.txt
View File

@ -1,63 +1,63 @@
# Minimum CMake required
cmake_minimum_required(VERSION 2.8.12)
# Project
project(protobuf-examples)
# Find required protobuf package
find_package(protobuf CONFIG REQUIRED)
if(protobuf_VERBOSE)
message(STATUS "Using Protocol Buffers ${Protobuf_VERSION}")
endif()
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
# http://www.cmake.org/Wiki/CMake_FAQ#How_can_I_build_my_MSVC_application_with_a_static_runtime.3F
if(MSVC AND protobuf_MSVC_STATIC_RUNTIME)
foreach(flag_var
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
if(${flag_var} MATCHES "/MD")
string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
endif(${flag_var} MATCHES "/MD")
endforeach()
endif()
foreach(example add_person list_people)
set(${example}_SRCS ${example}.cc)
set(${example}_PROTOS addressbook.proto)
#Code Generation
if(protobuf_MODULE_COMPATIBLE) #Legacy Support
protobuf_generate_cpp(${example}_PROTO_SRCS ${example}_PROTO_HDRS ${${example}_PROTOS})
list(APPEND ${example}_SRCS ${${example}_PROTO_SRCS} ${${example}_PROTO_HDRS})
else()
foreach(proto_file ${${example}_PROTOS})
get_filename_component(proto_file_abs ${proto_file} ABSOLUTE)
get_filename_component(basename ${proto_file} NAME_WE)
set(generated_files ${basename}.pb.cc ${basename}.pb.h)
list(APPEND ${example}_SRCS ${generated_files})
add_custom_command(
OUTPUT ${generated_files}
COMMAND protobuf::protoc
ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${CMAKE_CURRENT_SOURCE_DIR} ${proto_file_abs}
COMMENT "Generating ${generated_files} from ${proto_file}"
VERBATIM
)
endforeach()
endif()
#Executable setup
set(executable_name ${example}_cpp)
add_executable(${executable_name} ${${example}_SRCS} ${${example}_PROTOS})
if(protobuf_MODULE_COMPATIBLE) #Legacy mode
target_include_directories(${executable_name} PUBLIC ${PROTOBUF_INCLUDE_DIRS})
target_link_libraries(${executable_name} ${PROTOBUF_LIBRARIES})
else()
target_link_libraries(${executable_name} protobuf::libprotobuf)
endif()
endforeach()
# Minimum CMake required
cmake_minimum_required(VERSION 2.8.12)
# Project
project(protobuf-examples)
# Find required protobuf package
find_package(protobuf CONFIG REQUIRED)
if(protobuf_VERBOSE)
message(STATUS "Using Protocol Buffers ${Protobuf_VERSION}")
endif()
set(CMAKE_INCLUDE_CURRENT_DIR TRUE)
# http://www.cmake.org/Wiki/CMake_FAQ#How_can_I_build_my_MSVC_application_with_a_static_runtime.3F
if(MSVC AND protobuf_MSVC_STATIC_RUNTIME)
foreach(flag_var
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
if(${flag_var} MATCHES "/MD")
string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}")
endif(${flag_var} MATCHES "/MD")
endforeach()
endif()
foreach(example add_person list_people)
set(${example}_SRCS ${example}.cc)
set(${example}_PROTOS addressbook.proto)
#Code Generation
if(protobuf_MODULE_COMPATIBLE) #Legacy Support
protobuf_generate_cpp(${example}_PROTO_SRCS ${example}_PROTO_HDRS ${${example}_PROTOS})
list(APPEND ${example}_SRCS ${${example}_PROTO_SRCS} ${${example}_PROTO_HDRS})
else()
foreach(proto_file ${${example}_PROTOS})
get_filename_component(proto_file_abs ${proto_file} ABSOLUTE)
get_filename_component(basename ${proto_file} NAME_WE)
set(generated_files ${basename}.pb.cc ${basename}.pb.h)
list(APPEND ${example}_SRCS ${generated_files})
add_custom_command(
OUTPUT ${generated_files}
COMMAND protobuf::protoc
ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} -I ${CMAKE_CURRENT_SOURCE_DIR} ${proto_file_abs}
COMMENT "Generating ${generated_files} from ${proto_file}"
VERBATIM
)
endforeach()
endif()
#Executable setup
set(executable_name ${example}_cpp)
add_executable(${executable_name} ${${example}_SRCS} ${${example}_PROTOS})
if(protobuf_MODULE_COMPATIBLE) #Legacy mode
target_include_directories(${executable_name} PUBLIC ${PROTOBUF_INCLUDE_DIRS})
target_link_libraries(${executable_name} ${PROTOBUF_LIBRARIES})
else()
target_link_libraries(${executable_name} protobuf::libprotobuf)
endif()
endforeach()

38
java/core/src/main/java/com/google/protobuf/CodedOutputStream.java
View File

@ -144,6 +144,44 @@ public abstract class CodedOutputStream extends ByteOutput {
return new NioEncoder(byteBuffer);
}
/**
* Configures serialization to be deterministic.
*
* <p>The deterministic serialization guarantees that for a given binary, equal (defined by the
* {@code equals()} methods in protos) messages will always be serialized to the same bytes. This
* implies:
*
* <ul>
* <li>repeated serialization of a message will return the same bytes
* <li>different processes of the same binary (which may be executing on different machines) will
* serialize equal messages to the same bytes.
* </ul>
*
* <p>Note the deterministic serialization is NOT canonical across languages; it is also unstable
* across different builds with schema changes due to unknown fields. Users who need canonical
* serialization, e.g. persistent storage in a canonical form, fingerprinting, etc, should define
* their own canonicalization specification and implement the serializer using reflection APIs
* rather than relying on this API.
*
* <p> Once set, the serializer will: (Note this is an implementation detail and may subject to
* change in the future)
*
* <ul>
* <li> sort map entries by keys in lexicographical order or numerical order. Note: For string
* keys, the order is based on comparing the Unicode value of each character in the strings.
* The order may be different from the deterministic serialization in other languages where
* maps are sorted on the lexicographical order of the UTF8 encoded keys.
* </ul>
*/
public final void useDeterministicSerialization() {
serializationDeterministic = true;
}
boolean isSerializationDeterministic() {
return serializationDeterministic;
}
private boolean serializationDeterministic;
/**
* Create a new {@code CodedOutputStream} that writes to the given {@link ByteBuffer}.
*

8
java/core/src/main/java/com/google/protobuf/DynamicMessage.java
View File

@ -526,6 +526,14 @@ public final class DynamicMessage extends AbstractMessage {
fields.clearField(oldField);
}
oneofCases[index] = field;
} else if (field.getFile().getSyntax() == Descriptors.FileDescriptor.Syntax.PROTO3) {
if (!field.isRepeated()
&& field.getJavaType() != FieldDescriptor.JavaType.MESSAGE
&& value.equals(field.getDefaultValue())) {
// In proto3, setting a field to its default value is equivalent to clearing the field.
fields.clearField(field);
return this;
}
}
fields.setField(field, value);
return this;

50
java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java
View File

@ -31,6 +31,8 @@
package com.google.protobuf;
import com.google.protobuf.Descriptors.Descriptor;
import com.google.protobuf.Descriptors.EnumDescriptor;
import com.google.protobuf.Descriptors.EnumValueDescriptor;
import com.google.protobuf.Descriptors.FieldDescriptor;
import com.google.protobuf.FieldPresenceTestProto.TestAllTypes;
import com.google.protobuf.FieldPresenceTestProto.TestOptionalFieldsOnly;
@ -253,6 +255,54 @@ public class FieldPresenceTest extends TestCase {
assertEquals(4, message.getAllFields().size());
}
public void testFieldPresenceDynamicMessage() {
Descriptor descriptor = TestAllTypes.getDescriptor();
FieldDescriptor optionalInt32Field = descriptor.findFieldByName("optional_int32");
FieldDescriptor optionalStringField = descriptor.findFieldByName("optional_string");
FieldDescriptor optionalBytesField = descriptor.findFieldByName("optional_bytes");
FieldDescriptor optionalNestedEnumField = descriptor.findFieldByName("optional_nested_enum");
EnumDescriptor enumDescriptor = optionalNestedEnumField.getEnumType();
EnumValueDescriptor defaultEnumValueDescriptor = enumDescriptor.getValues().get(0);
EnumValueDescriptor nonDefaultEnumValueDescriptor = enumDescriptor.getValues().get(1);
DynamicMessage defaultInstance = DynamicMessage.getDefaultInstance(descriptor);
// Field not present.
DynamicMessage message = defaultInstance.newBuilderForType().build();
assertFalse(message.hasField(optionalInt32Field));
assertFalse(message.hasField(optionalStringField));
assertFalse(message.hasField(optionalBytesField));
assertFalse(message.hasField(optionalNestedEnumField));
assertEquals(0, message.getAllFields().size());
// Field set to non-default value is seen as present.
message =
defaultInstance
.newBuilderForType()
.setField(optionalInt32Field, 1)
.setField(optionalStringField, "x")
.setField(optionalBytesField, ByteString.copyFromUtf8("y"))
.setField(optionalNestedEnumField, nonDefaultEnumValueDescriptor)
.build();
assertTrue(message.hasField(optionalInt32Field));
assertTrue(message.hasField(optionalStringField));
assertTrue(message.hasField(optionalBytesField));
assertTrue(message.hasField(optionalNestedEnumField));
assertEquals(4, message.getAllFields().size());
// Field set to default value is seen as not present.
message = message.toBuilder()
.setField(optionalInt32Field, 0)
.setField(optionalStringField, "")
.setField(optionalBytesField, ByteString.EMPTY)
.setField(optionalNestedEnumField, defaultEnumValueDescriptor)
.build();
assertFalse(message.hasField(optionalInt32Field));
assertFalse(message.hasField(optionalStringField));
assertFalse(message.hasField(optionalBytesField));
assertFalse(message.hasField(optionalNestedEnumField));
assertEquals(0, message.getAllFields().size());
}
public void testMessageField() {
TestAllTypes.Builder builder = TestAllTypes.newBuilder();
assertFalse(builder.hasOptionalNestedMessage());

3
java/core/src/test/java/com/google/protobuf/LazyStringArrayListTest.java
View File

@ -35,6 +35,7 @@ import static java.util.Arrays.asList;
import junit.framework.TestCase;
import java.util.ArrayList;
import java.util.Collections;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.List;
@ -233,7 +234,7 @@ public class LazyStringArrayListTest extends TestCase {
}
try {
list.addAllByteArray(asList(BYTE_STRING_A.toByteArray()));
list.addAllByteArray(Collections.singletonList(BYTE_STRING_A.toByteArray()));
fail();
} catch (UnsupportedOperationException e) {
// expected

40
java/util/src/main/java/com/google/protobuf/util/JsonFormat.java
View File

@ -116,7 +116,8 @@ public class JsonFormat {
private Printer(
TypeRegistry registry,
boolean includingDefaultValueFields,
boolean preservingProtoFieldNames, boolean omittingInsignificantWhitespace) {
boolean preservingProtoFieldNames,
boolean omittingInsignificantWhitespace) {
this.registry = registry;
this.includingDefaultValueFields = includingDefaultValueFields;
this.preservingProtoFieldNames = preservingProtoFieldNames;
@ -133,7 +134,11 @@ public class JsonFormat {
if (this.registry != TypeRegistry.getEmptyTypeRegistry()) {
throw new IllegalArgumentException("Only one registry is allowed.");
}
return new Printer(registry, includingDefaultValueFields, preservingProtoFieldNames, omittingInsignificantWhitespace);
return new Printer(
registry,
includingDefaultValueFields,
preservingProtoFieldNames,
omittingInsignificantWhitespace);
}
/**
@ -143,7 +148,8 @@ public class JsonFormat {
* {@link Printer}.
*/
public Printer includingDefaultValueFields() {
return new Printer(registry, true, preservingProtoFieldNames, omittingInsignificantWhitespace);
return new Printer(
registry, true, preservingProtoFieldNames, omittingInsignificantWhitespace);
}
/**
@ -153,7 +159,8 @@ public class JsonFormat {
* current {@link Printer}.
*/
public Printer preservingProtoFieldNames() {
return new Printer(registry, includingDefaultValueFields, true, omittingInsignificantWhitespace);
return new Printer(
registry, includingDefaultValueFields, true, omittingInsignificantWhitespace);
}
@ -172,7 +179,7 @@ public class JsonFormat {
* See <a href="https://tools.ietf.org/html/rfc7159">https://tools.ietf.org/html/rfc7159</a>
* current {@link Printer}.
*/
public Printer omittingInsignificantWhitespace(){
public Printer omittingInsignificantWhitespace() {
return new Printer(registry, includingDefaultValueFields, preservingProtoFieldNames, true);
}
@ -186,7 +193,12 @@ public class JsonFormat {
public void appendTo(MessageOrBuilder message, Appendable output) throws IOException {
// TODO(xiaofeng): Investigate the allocation overhead and optimize for
// mobile.
new PrinterImpl(registry, includingDefaultValueFields, preservingProtoFieldNames, output, omittingInsignificantWhitespace)
new PrinterImpl(
registry,
includingDefaultValueFields,
preservingProtoFieldNames,
output,
omittingInsignificantWhitespace)
.print(message);
}
@ -379,18 +391,18 @@ public class JsonFormat {
*/
interface TextGenerator {
void indent();
void outdent();
void print(final CharSequence text) throws IOException;
}
/**
* Format the json without indentation
*/
private static final class CompactTextGenerator implements TextGenerator{
private static final class CompactTextGenerator implements TextGenerator {
private final Appendable output;
private CompactTextGenerator(final Appendable output) {
this.output = output;
}
@ -411,12 +423,11 @@ public class JsonFormat {
public void print(final CharSequence text) throws IOException {
output.append(text);
}
}
/**
* A TextGenerator adds indentation when writing formatted text.
*/
private static final class PrettyTextGenerator implements TextGenerator{
private static final class PrettyTextGenerator implements TextGenerator {
private final Appendable output;
private final StringBuilder indent = new StringBuilder();
private boolean atStartOfLine = true;
@ -496,7 +507,8 @@ public class JsonFormat {
TypeRegistry registry,
boolean includingDefaultValueFields,
boolean preservingProtoFieldNames,
Appendable jsonOutput, boolean omittingInsignificantWhitespace) {
Appendable jsonOutput,
boolean omittingInsignificantWhitespace) {
this.registry = registry;
this.includingDefaultValueFields = includingDefaultValueFields;
this.preservingProtoFieldNames = preservingProtoFieldNames;
@ -734,9 +746,7 @@ public class JsonFormat {
}
/** Prints a regular message with an optional type URL. */
private void print(MessageOrBuilder message, String typeUrl)
throws IOException {
private void print(MessageOrBuilder message, String typeUrl) throws IOException {
generator.print("{" + blankOrNewLine);
generator.indent();

22
java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java
View File

@ -140,7 +140,7 @@ public class JsonFormatTest extends TestCase {
private String toJsonString(Message message) throws IOException {
return JsonFormat.printer().print(message);
}
private String toCompactJsonString(Message message) throws IOException{
private String toCompactJsonString(Message message) throws IOException {
return JsonFormat.printer().omittingInsignificantWhitespace().print(message);
}
@ -1172,7 +1172,9 @@ public class JsonFormatTest extends TestCase {
public void testOmittingInsignificantWhiteSpace() throws Exception {
TestAllTypes message = TestAllTypes.newBuilder().setOptionalInt32(12345).build();
assertEquals("{" + "\"optionalInt32\":12345" + "}", JsonFormat.printer().omittingInsignificantWhitespace().print(message));
assertEquals(
"{" + "\"optionalInt32\":12345" + "}",
JsonFormat.printer().omittingInsignificantWhitespace().print(message));
TestAllTypes message1 = TestAllTypes.getDefaultInstance();
assertEquals("{}", JsonFormat.printer().omittingInsignificantWhitespace().print(message1));
TestAllTypes.Builder builder = TestAllTypes.newBuilder();
@ -1224,4 +1226,20 @@ public class JsonFormatTest extends TestCase {
toCompactJsonString(message2));
}
// Regression test for b/29892357
public void testEmptyWrapperTypesInAny() throws Exception {
JsonFormat.TypeRegistry registry =
JsonFormat.TypeRegistry.newBuilder().add(TestAllTypes.getDescriptor()).build();
JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(registry);
Any.Builder builder = Any.newBuilder();
parser.merge(
"{\n"
+ " \"@type\": \"type.googleapis.com/google.protobuf.BoolValue\",\n"
+ " \"value\": false\n"
+ "}\n",
builder);
Any any = builder.build();
assertEquals(0, any.getValue().size());
}
}

46
js/message_test.js
View File

@ -215,6 +215,10 @@ describe('Message test suite', function() {
assertEquals(true, response.getBoolField());
assertEquals(11, response.getIntField());
assertEquals(13, response.getEnumField());
assertFalse(response.hasStringField());
assertFalse(response.hasBoolField());
assertFalse(response.hasIntField());
assertFalse(response.hasEnumField());
// Test with null values, as would be returned by a JSON serializer.
response = makeDefault([null, null, null, null]);
@ -222,6 +226,10 @@ describe('Message test suite', function() {
assertEquals(true, response.getBoolField());
assertEquals(11, response.getIntField());
assertEquals(13, response.getEnumField());
assertFalse(response.hasStringField());
assertFalse(response.hasBoolField());
assertFalse(response.hasIntField());
assertFalse(response.hasEnumField());
// Test with false-like values.
response = makeDefault(['', false, 0, 0]);
@ -229,6 +237,10 @@ describe('Message test suite', function() {
assertEquals(false, response.getBoolField());
assertEquals(true, response.getIntField() == 0);
assertEquals(true, response.getEnumField() == 0);
assertTrue(response.hasStringField());
assertTrue(response.hasBoolField());
assertTrue(response.hasIntField());
assertTrue(response.hasEnumField());
// Test that clearing the values reverts them to the default state.
response = makeDefault(['blah', false, 111, 77]);
@ -238,6 +250,10 @@ describe('Message test suite', function() {
assertEquals(true, response.getBoolField());
assertEquals(11, response.getIntField());
assertEquals(13, response.getEnumField());
assertFalse(response.hasStringField());
assertFalse(response.hasBoolField());
assertFalse(response.hasIntField());
assertFalse(response.hasEnumField());
// Test that setFoo(null) clears the values.
response = makeDefault(['blah', false, 111, 77]);
@ -247,6 +263,10 @@ describe('Message test suite', function() {
assertEquals(true, response.getBoolField());
assertEquals(11, response.getIntField());
assertEquals(13, response.getEnumField());
assertFalse(response.hasStringField());
assertFalse(response.hasBoolField());
assertFalse(response.hasIntField());
assertFalse(response.hasEnumField());
});
it('testMessageRegistration', function() {
@ -269,6 +289,8 @@ describe('Message test suite', function() {
assertUndefined(foo.getAString());
assertUndefined(foo.getABool());
assertUndefined(foo.getANestedMessage());
assertFalse(foo.hasAString());
assertFalse(foo.hasABool());
assertObjectEquals([], foo.getARepeatedMessageList());
assertObjectEquals([], foo.getARepeatedStringList());
// NOTE: We want the missing fields in 'expected' to be undefined,
@ -291,6 +313,8 @@ describe('Message test suite', function() {
assertNull(foo.getAString());
assertNull(foo.getABool());
assertNull(foo.getANestedMessage());
assertFalse(foo.hasAString());
assertFalse(foo.hasABool());
assertObjectEquals([], foo.getARepeatedMessageList());
assertObjectEquals([], foo.getARepeatedStringList());
assertObjectEquals([null, null, null, [], []], foo.toArray());
@ -307,6 +331,8 @@ describe('Message test suite', function() {
assertUndefined(foo.getAString());
assertUndefined(foo.getABool());
assertUndefined(foo.getANestedMessage());
assertFalse(foo.hasAString());
assertFalse(foo.hasABool());
assertObjectEquals([], foo.getARepeatedMessageList());
assertObjectEquals([], foo.getARepeatedStringList());
expected = [,,, [], []];
@ -800,14 +826,20 @@ describe('Message test suite', function() {
var message = new proto.jspb.test.TestMessageWithOneof;
assertUndefined(message.getPone());
assertUndefined(message.getPthree());
assertFalse(message.hasPone());
assertFalse(message.hasPthree());
message.setPone('hi');
assertEquals('hi', message.getPone());
assertUndefined(message.getPthree());
assertTrue(message.hasPone());
assertFalse(message.hasPthree());
message.setPthree('bye');
assertUndefined(message.getPone());
assertEquals('bye', message.getPthree());
assertFalse(message.hasPone());
assertTrue(message.hasPthree());
});
it('testSettingOneofFieldDoesNotClearFieldsFromOtherUnions', function() {
@ -816,17 +848,23 @@ describe('Message test suite', function() {
assertUndefined(message.getPone());
assertUndefined(message.getPthree());
assertUndefined(message.getRone());
assertFalse(message.hasPone());
assertFalse(message.hasPthree());
message.setPone('hi');
message.setRone(other);
assertEquals('hi', message.getPone());
assertUndefined(message.getPthree());
assertEquals(other, message.getRone());
assertTrue(message.hasPone());
assertFalse(message.hasPthree());
message.setPthree('bye');
assertUndefined(message.getPone());
assertEquals('bye', message.getPthree());
assertEquals(other, message.getRone());
assertFalse(message.hasPone());
assertTrue(message.hasPthree());
});
it('testUnsetsOneofCaseWhenFieldIsCleared', function() {
@ -884,6 +922,8 @@ describe('Message test suite', function() {
var message = new proto.jspb.test.TestMessageWithOneof;
assertUndefined(message.getBone());
assertEquals(1234, message.getBtwo());
assertFalse(message.hasBone());
assertFalse(message.hasBtwo());
assertEquals(
proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase
.DEFAULT_ONEOF_B_NOT_SET,
@ -892,12 +932,16 @@ describe('Message test suite', function() {
message.setBone(2);
assertEquals(2, message.getBone());
assertEquals(1234, message.getBtwo());
assertTrue(message.hasBone());
assertFalse(message.hasBtwo());
assertEquals(
proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase.BONE,
message.getDefaultOneofBCase());
message.setBtwo(3);
assertUndefined(message.getBone());
assertFalse(message.hasBone());
assertTrue(message.hasBtwo());
assertEquals(3, message.getBtwo());
assertEquals(
proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase.BTWO,
@ -905,6 +949,8 @@ describe('Message test suite', function() {
message.clearBtwo();
assertUndefined(message.getBone());
assertFalse(message.hasBone());
assertFalse(message.hasBtwo());
assertEquals(1234, message.getBtwo());
assertEquals(
proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase

15
js/proto3_test.js
View File

@ -225,12 +225,18 @@ describe('proto3Test', function() {
assertEquals(msg.getOneofForeignMessage(), undefined);
assertEquals(msg.getOneofString(), undefined);
assertEquals(msg.getOneofBytes(), undefined);
assertFalse(msg.hasOneofUint32());
assertFalse(msg.hasOneofString());
assertFalse(msg.hasOneofBytes());
msg.setOneofUint32(42);
assertEquals(msg.getOneofUint32(), 42);
assertEquals(msg.getOneofForeignMessage(), undefined);
assertEquals(msg.getOneofString(), undefined);
assertEquals(msg.getOneofBytes(), undefined);
assertTrue(msg.hasOneofUint32());
assertFalse(msg.hasOneofString());
assertFalse(msg.hasOneofBytes());
var submsg = new proto.jspb.test.ForeignMessage();
@ -239,12 +245,18 @@ describe('proto3Test', function() {
assertEquals(msg.getOneofForeignMessage(), submsg);
assertEquals(msg.getOneofString(), undefined);
assertEquals(msg.getOneofBytes(), undefined);
assertFalse(msg.hasOneofUint32());
assertFalse(msg.hasOneofString());
assertFalse(msg.hasOneofBytes());
msg.setOneofString('hello');
assertEquals(msg.getOneofUint32(), undefined);
assertEquals(msg.getOneofForeignMessage(), undefined);
assertEquals(msg.getOneofString(), 'hello');
assertEquals(msg.getOneofBytes(), undefined);
assertFalse(msg.hasOneofUint32());
assertTrue(msg.hasOneofString());
assertFalse(msg.hasOneofBytes());
msg.setOneofBytes(goog.crypt.base64.encodeString('\u00FF\u00FF'));
assertEquals(msg.getOneofUint32(), undefined);
@ -252,6 +264,9 @@ describe('proto3Test', function() {
assertEquals(msg.getOneofString(), undefined);
assertEquals(msg.getOneofBytes_asB64(),
goog.crypt.base64.encodeString('\u00FF\u00FF'));
assertFalse(msg.hasOneofUint32());
assertFalse(msg.hasOneofString());
assertTrue(msg.hasOneofBytes());
});

1
js/test.proto
View File

@ -233,3 +233,4 @@ message TestEndsWithBytes {
optional int32 value = 1;
optional bytes data = 2;
}

6
python/google/protobuf/internal/json_format_test.py
View File

@ -252,10 +252,7 @@ class JsonFormatTest(JsonFormatBase):
message = json_format_proto3_pb2.TestMessage()
json_format.Parse('{"stringValue": "\\uD83D\\uDE01"}', message)
self.assertEqual(message.string_value,
b'\xF0\x9F\x98\x81'.decode("utf-8", "strict"))
# TODO: add test that UTF-8 encoded surrogate code points are rejected.
# UTF-8 does not allow them.
b'\xF0\x9F\x98\x81'.decode('utf-8', 'strict'))
# Error case: unpaired high surrogate.
self.CheckError(
@ -267,7 +264,6 @@ class JsonFormatTest(JsonFormatBase):
'{"stringValue": "\\uDE01"}',
r'Invalid \\uXXXX escape|Unpaired.*surrogate')
def testTimestampMessage(self):
message = json_format_proto3_pb2.TestTimestamp()
message.value.seconds = 0

26
python/google/protobuf/internal/python_message.py
View File

@ -76,7 +76,6 @@ from google.protobuf.internal import well_known_types
from google.protobuf.internal import wire_format
from google.protobuf import descriptor as descriptor_mod
from google.protobuf import message as message_mod
from google.protobuf import symbol_database
from google.protobuf import text_format
_FieldDescriptor = descriptor_mod.FieldDescriptor
@ -98,16 +97,12 @@ class GeneratedProtocolMessageType(type):
classes at runtime, as in this example:
mydescriptor = Descriptor(.....)
class MyProtoClass(Message):
__metaclass__ = GeneratedProtocolMessageType
DESCRIPTOR = mydescriptor
factory = symbol_database.Default()
factory.pool.AddDescriptor(mydescriptor)
MyProtoClass = factory.GetPrototype(mydescriptor)
myproto_instance = MyProtoClass()
myproto.foo_field = 23
...
The above example will not work for nested types. If you wish to include them,
use reflection.MakeClass() instead of manually instantiating the class in
order to create the appropriate class structure.
"""
# Must be consistent with the protocol-compiler code in
@ -926,26 +921,33 @@ def _InternalUnpackAny(msg):
Returns:
The unpacked message.
"""
# TODO(amauryfa): Don't use the factory of generated messages.
# To make Any work with custom factories, use the message factory of the
# parent message.
# pylint: disable=g-import-not-at-top
from google.protobuf import symbol_database
factory = symbol_database.Default()
type_url = msg.type_url
db = symbol_database.Default()
if not type_url:
return None
# TODO(haberman): For now we just strip the hostname. Better logic will be
# required.
type_name = type_url.split("/")[-1]
descriptor = db.pool.FindMessageTypeByName(type_name)
type_name = type_url.split('/')[-1]
descriptor = factory.pool.FindMessageTypeByName(type_name)
if descriptor is None:
return None
message_class = db.GetPrototype(descriptor)
message_class = factory.GetPrototype(descriptor)
message = message_class()
message.ParseFromString(msg.value)
return message
def _AddEqualsMethod(message_descriptor, cls):
"""Helper for _AddMessageMethods()."""
def __eq__(self, other):

1
python/google/protobuf/internal/reflection_test.py
View File

@ -972,6 +972,7 @@ class ReflectionTest(unittest.TestCase):
proto.repeated_nested_message.add(bb=23)
self.assertEqual(1, len(proto.repeated_nested_message))
self.assertEqual(23, proto.repeated_nested_message[0].bb)
self.assertRaises(TypeError, proto.repeated_nested_message.add, 23)
def testRepeatedCompositeRemove(self):
proto = unittest_pb2.TestAllTypes()

26
python/google/protobuf/internal/symbol_database_test.py
View File

@ -39,26 +39,28 @@ except ImportError:
from google.protobuf import unittest_pb2
from google.protobuf import descriptor
from google.protobuf import descriptor_pool
from google.protobuf import symbol_database
class SymbolDatabaseTest(unittest.TestCase):
def _Database(self):
# TODO(b/17734095): Remove this difference when the C++ implementation
# supports multiple databases.
if descriptor._USE_C_DESCRIPTORS:
return symbol_database.Default()
# The C++ implementation does not allow mixing descriptors from
# different pools.
db = symbol_database.SymbolDatabase(pool=descriptor_pool.Default())
else:
db = symbol_database.SymbolDatabase()
# Register representative types from unittest_pb2.
db.RegisterFileDescriptor(unittest_pb2.DESCRIPTOR)
db.RegisterMessage(unittest_pb2.TestAllTypes)
db.RegisterMessage(unittest_pb2.TestAllTypes.NestedMessage)
db.RegisterMessage(unittest_pb2.TestAllTypes.OptionalGroup)
db.RegisterMessage(unittest_pb2.TestAllTypes.RepeatedGroup)
db.RegisterEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR)
db.RegisterEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR)
return db
# Register representative types from unittest_pb2.
db.RegisterFileDescriptor(unittest_pb2.DESCRIPTOR)
db.RegisterMessage(unittest_pb2.TestAllTypes)
db.RegisterMessage(unittest_pb2.TestAllTypes.NestedMessage)
db.RegisterMessage(unittest_pb2.TestAllTypes.OptionalGroup)
db.RegisterMessage(unittest_pb2.TestAllTypes.RepeatedGroup)
db.RegisterEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR)
db.RegisterEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR)
return db
def testGetPrototype(self):
instance = self._Database().GetPrototype(

6
python/google/protobuf/pyext/cpp_message.py
View File

@ -48,9 +48,9 @@ class GeneratedProtocolMessageType(_message.MessageMeta):
classes at runtime, as in this example:
mydescriptor = Descriptor(.....)
class MyProtoClass(Message):
__metaclass__ = GeneratedProtocolMessageType
DESCRIPTOR = mydescriptor
factory = symbol_database.Default()
factory.pool.AddDescriptor(mydescriptor)
MyProtoClass = factory.GetPrototype(mydescriptor)
myproto_instance = MyProtoClass()
myproto.foo_field = 23
...

3
python/google/protobuf/pyext/map_container.cc
View File

@ -348,9 +348,10 @@ PyObject* MapReflectionFriend::Contains(PyObject* _self, PyObject* key) {
}
// Initializes the underlying Message object of "to" so it becomes a new parent
// repeated scalar, and copies all the values from "from" to it. A child scalar
// map container, and copies all the values from "from" to it. A child map
// container can be released by passing it as both from and to (e.g. making it
// the recipient of the new parent message and copying the values from itself).
// In fact, this is the only supported use at the moment.
static int InitializeAndCopyToParentContainer(MapContainer* from,
MapContainer* to) {
// For now we require from == to, re-evaluate if we want to support deep copy

16
python/google/protobuf/pyext/message.cc
View File

@ -1041,7 +1041,12 @@ int InternalDeleteRepeatedField(
}
// Initializes fields of a message. Used in constructors.
int InitAttributes(CMessage* self, PyObject* kwargs) {
int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) {
if (args != NULL && PyTuple_Size(args) != 0) {
PyErr_SetString(PyExc_TypeError, "No positional arguments allowed");
return -1;
}
if (kwargs == NULL) {
return 0;
}
@ -1167,7 +1172,7 @@ int InitAttributes(CMessage* self, PyObject* kwargs) {
}
CMessage* cmessage = reinterpret_cast<CMessage*>(message.get());
if (PyDict_Check(value)) {
if (InitAttributes(cmessage, value) < 0) {
if (InitAttributes(cmessage, NULL, value) < 0) {
return -1;
}
} else {
@ -1245,12 +1250,7 @@ static PyObject* New(PyTypeObject* cls,
// The __init__ method of Message classes.
// It initializes fields from keywords passed to the constructor.
static int Init(CMessage* self, PyObject* args, PyObject* kwargs) {
if (PyTuple_Size(args) != 0) {
PyErr_SetString(PyExc_TypeError, "No positional arguments allowed");
return -1;
}
return InitAttributes(self, kwargs);
return InitAttributes(self, args, kwargs);
}
// ---------------------------------------------------------------------

4
python/google/protobuf/pyext/message.h
View File

@ -237,7 +237,9 @@ PyObject* HasFieldByDescriptor(
PyObject* HasField(CMessage* self, PyObject* arg);
// Initializes values of fields on a newly constructed message.
int InitAttributes(CMessage* self, PyObject* kwargs);
// Note that positional arguments are disallowed: 'args' must be NULL or the
// empty tuple.
int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs);
PyObject* MergeFrom(CMessage* self, PyObject* arg);

4
python/google/protobuf/pyext/repeated_composite_container.cc
View File

@ -146,7 +146,7 @@ static PyObject* AddToAttached(RepeatedCompositeContainer* self,
cmsg->owner = self->owner;
cmsg->message = sub_message;
cmsg->parent = self->parent;
if (cmessage::InitAttributes(cmsg, kwargs) < 0) {
if (cmessage::InitAttributes(cmsg, args, kwargs) < 0) {
Py_DECREF(cmsg);
return NULL;
}
@ -166,7 +166,7 @@ static PyObject* AddToReleased(RepeatedCompositeContainer* self,
// Create a new Message detached from the rest.
PyObject* py_cmsg = PyEval_CallObjectWithKeywords(
self->child_message_class->AsPyObject(), NULL, kwargs);
self->child_message_class->AsPyObject(), args, kwargs);
if (py_cmsg == NULL)
return NULL;

8
python/google/protobuf/reflection.py
View File

@ -58,13 +58,7 @@ else:
from google.protobuf.internal import python_message as message_impl
# The type of all Message classes.
# Part of the public interface.
#
# Used by generated files, but clients can also use it at runtime:
# mydescriptor = pool.FindDescriptor(.....)
# class MyProtoClass(Message):
# __metaclass__ = GeneratedProtocolMessageType
# DESCRIPTOR = mydescriptor
# Part of the public interface, but normally only used by message factories.
GeneratedProtocolMessageType = message_impl.GeneratedProtocolMessageType

82
python/google/protobuf/symbol_database.py
View File

@ -30,11 +30,9 @@
"""A database of Python protocol buffer generated symbols.
SymbolDatabase makes it easy to create new instances of a registered type, given
only the type's protocol buffer symbol name. Once all symbols are registered,
they can be accessed using either the MessageFactory interface which
SymbolDatabase exposes, or the DescriptorPool interface of the underlying
pool.
SymbolDatabase is the MessageFactory for messages generated at compile time,
and makes it easy to create new instances of a registered type, given only the
type's protocol buffer symbol name.
Example usage:
@ -61,27 +59,17 @@ Example usage:
from google.protobuf import descriptor_pool
from google.protobuf import message_factory
class SymbolDatabase(object):
"""A database of Python generated symbols.
SymbolDatabase also models message_factory.MessageFactory.
The symbol database can be used to keep a global registry of all protocol
buffer types used within a program.
"""
def __init__(self, pool=None):
"""Constructor."""
self._symbols = {}
self._symbols_by_file = {}
self.pool = pool or descriptor_pool.Default()
class SymbolDatabase(message_factory.MessageFactory):
"""A database of Python generated symbols."""
def RegisterMessage(self, message):
"""Registers the given message type in the local database.
Calls to GetSymbol() and GetMessages() will return messages registered here.
Args:
message: a message.Message, to be registered.
@ -90,10 +78,7 @@ class SymbolDatabase(object):
"""
desc = message.DESCRIPTOR
self._symbols[desc.full_name] = message
if desc.file.name not in self._symbols_by_file:
self._symbols_by_file[desc.file.name] = {}
self._symbols_by_file[desc.file.name][desc.full_name] = message
self._classes[desc.full_name] = message
self.pool.AddDescriptor(desc)
return message
@ -136,47 +121,46 @@ class SymbolDatabase(object):
KeyError: if the symbol could not be found.
"""
return self._symbols[symbol]
def GetPrototype(self, descriptor):
"""Builds a proto2 message class based on the passed in descriptor.
Passing a descriptor with a fully qualified name matching a previous
invocation will cause the same class to be returned.
Args:
descriptor: The descriptor to build from.
Returns:
A class describing the passed in descriptor.
"""
return self.GetSymbol(descriptor.full_name)
return self._classes[symbol]
def GetMessages(self, files):
"""Gets all the messages from a specified file.
This will find and resolve dependencies, failing if they are not registered
in the symbol database.
# TODO(amauryfa): Fix the differences with MessageFactory.
"""Gets all registered messages from a specified file.
Only messages already created and registered will be returned; (this is the
case for imported _pb2 modules)
But unlike MessageFactory, this version also returns nested messages.
Args:
files: The file names to extract messages from.
Returns:
A dictionary mapping proto names to the message classes. This will include
any dependent messages as well as any messages defined in the same file as
a specified message.
A dictionary mapping proto names to the message classes.
Raises:
KeyError: if a file could not be found.
"""
def _GetAllMessageNames(desc):
"""Walk a message Descriptor and recursively yields all message names."""
yield desc.full_name
for msg_desc in desc.nested_types:
for full_name in _GetAllMessageNames(msg_desc):
yield full_name
result = {}
for f in files:
result.update(self._symbols_by_file[f])
for file_name in files:
file_desc = self.pool.FindFileByName(file_name)
for msg_desc in file_desc.message_types_by_name.values():
for full_name in _GetAllMessageNames(msg_desc):
try:
result[full_name] = self._classes[full_name]
except KeyError:
# This descriptor has no registered class, skip it.
pass
return result
_DEFAULT = SymbolDatabase(pool=descriptor_pool.Default())

26
src/google/protobuf/compiler/cpp/cpp_file.cc
View File

@ -336,19 +336,6 @@ void FileGenerator::GenerateSource(io::Printer* printer) {
// Generate classes.
for (int i = 0; i < file_->message_type_count(); i++) {
if (i == 0 && HasGeneratedMethods(file_, options_)) {
printer->Print(
"\n"
"namespace {\n"
"\n"
"static void MergeFromFail(int line) GOOGLE_ATTRIBUTE_COLD;\n"
"static void MergeFromFail(int line) {\n"
" GOOGLE_CHECK(false) << __FILE__ << \":\" << line;\n"
"}\n"
"\n"
"} // namespace\n"
"\n");
}
printer->Print("\n");
printer->Print(kThickSeparator);
printer->Print("\n");
@ -464,9 +451,10 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) {
// and we only use AddDescriptors() to allocate default instances.
if (HasDescriptorMethods(file_, options_)) {
printer->Print(
"\n"
"void $assigndescriptorsname$() {\n",
"assigndescriptorsname", GlobalAssignDescriptorsName(file_->name()));
"\n"
"void $assigndescriptorsname$() GOOGLE_ATTRIBUTE_COLD;\n"
"void $assigndescriptorsname$() {\n",
"assigndescriptorsname", GlobalAssignDescriptorsName(file_->name()));
printer->Indent();
// Make sure the file has found its way into the pool. If a descriptor
@ -525,8 +513,9 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) {
// protobuf_RegisterTypes(): Calls
// MessageFactory::InternalRegisterGeneratedType() for each message type.
printer->Print(
"void protobuf_RegisterTypes(const ::std::string&) {\n"
" protobuf_AssignDescriptorsOnce();\n");
"void protobuf_RegisterTypes(const ::std::string&) GOOGLE_ATTRIBUTE_COLD;\n"
"void protobuf_RegisterTypes(const ::std::string&) {\n"
" protobuf_AssignDescriptorsOnce();\n");
printer->Indent();
for (int i = 0; i < file_->message_type_count(); i++) {
@ -566,6 +555,7 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) {
// Note that we don't need any special synchronization in the following
// code
// because it is called at static init time before any threads exist.
"void $adddescriptorsname$() GOOGLE_ATTRIBUTE_COLD;\n"
"void $adddescriptorsname$() {\n"
" static bool already_here = false;\n"
" if (already_here) return;\n"

219
src/google/protobuf/compiler/cpp/cpp_map_field.cc
View File

@ -251,117 +251,148 @@ GenerateMergeFromCodedStream(io::Printer* printer) const {
}
}
static void GenerateSerializationLoop(io::Printer* printer,
const map<string, string>& variables,
bool supports_arenas,
const string& utf8_check,
const string& loop_header,
const string& ptr,
bool loop_via_iterators) {
printer->Print(variables,
StrCat("::google::protobuf::scoped_ptr<$map_classname$> entry;\n",
loop_header, " {\n").c_str());
printer->Indent();
printer->Print(variables, StrCat(
"entry.reset($name$_.New$wrapper$(\n"
" ", ptr, "->first, ", ptr, "->second));\n"
"$write_entry$;\n").c_str());
// If entry is allocated by arena, its desctructor should be avoided.
if (supports_arenas) {
printer->Print(
"if (entry->GetArena() != NULL) {\n"
" entry.release();\n"
"}\n");
}
if (!utf8_check.empty()) {
// If loop_via_iterators is true then ptr is actually an iterator, and we
// create a pointer by prefixing it with "&*".
printer->Print(
StrCat(utf8_check, "(", (loop_via_iterators ? "&*" : ""), ptr, ");\n")
.c_str());
}
printer->Outdent();
printer->Print(
"}\n");
}
void MapFieldGenerator::
GenerateSerializeWithCachedSizes(io::Printer* printer) const {
printer->Print(variables_,
"{\n"
" ::google::protobuf::scoped_ptr<$map_classname$> entry;\n"
" for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n"
" it = this->$name$().begin();\n"
" it != this->$name$().end(); ++it) {\n");
// If entry is allocated by arena, its desctructor should be avoided.
if (SupportsArenas(descriptor_)) {
printer->Print(variables_,
" if (entry.get() != NULL && entry->GetArena() != NULL) {\n"
" entry.release();\n"
" }\n");
}
printer->Print(variables_,
" entry.reset($name$_.New$wrapper$(it->first, it->second));\n"
" ::google::protobuf::internal::WireFormatLite::Write$stream_writer$(\n"
" $number$, *entry, output);\n");
printer->Indent();
printer->Indent();
const FieldDescriptor* key_field =
descriptor_->message_type()->FindFieldByName("key");
const FieldDescriptor* value_field =
descriptor_->message_type()->FindFieldByName("value");
if (key_field->type() == FieldDescriptor::TYPE_STRING) {
GenerateUtf8CheckCodeForString(key_field, options_, false, variables_,
"it->first.data(), it->first.length(),\n",
printer);
}
if (value_field->type() == FieldDescriptor::TYPE_STRING) {
GenerateUtf8CheckCodeForString(value_field, options_, false, variables_,
"it->second.data(), it->second.length(),\n",
printer);
}
printer->Outdent();
printer->Outdent();
printer->Print(
" }\n");
// If entry is allocated by arena, its desctructor should be avoided.
if (SupportsArenas(descriptor_)) {
printer->Print(variables_,
" if (entry.get() != NULL && entry->GetArena() != NULL) {\n"
" entry.release();\n"
" }\n");
}
printer->Print("}\n");
map<string, string> variables(variables_);
variables["write_entry"] = "::google::protobuf::internal::WireFormatLite::Write" +
variables["stream_writer"] + "(\n " +
variables["number"] + ", *entry, output)";
variables["deterministic"] = "output->IsSerializationDeterminstic()";
GenerateSerializeWithCachedSizes(printer, variables);
}
void MapFieldGenerator::
GenerateSerializeWithCachedSizesToArray(io::Printer* printer) const {
printer->Print(variables_,
"{\n"
" ::google::protobuf::scoped_ptr<$map_classname$> entry;\n"
" for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n"
" it = this->$name$().begin();\n"
" it != this->$name$().end(); ++it) {\n");
// If entry is allocated by arena, its desctructor should be avoided.
if (SupportsArenas(descriptor_)) {
printer->Print(variables_,
" if (entry.get() != NULL && entry->GetArena() != NULL) {\n"
" entry.release();\n"
" }\n");
}
printer->Print(variables_,
" entry.reset($name$_.New$wrapper$(it->first, it->second));\n"
" target = ::google::protobuf::internal::WireFormatLite::\n"
" InternalWrite$declared_type$NoVirtualToArray(\n"
" $number$, *entry, false, target);\n");
map<string, string> variables(variables_);
variables["write_entry"] =
"target = ::google::protobuf::internal::WireFormatLite::\n"
" InternalWrite" + variables["declared_type"] +
"NoVirtualToArray(\n " + variables["number"] +
", *entry, deterministic, target);\n";
variables["deterministic"] = "deterministic";
GenerateSerializeWithCachedSizes(printer, variables);
}
void MapFieldGenerator::GenerateSerializeWithCachedSizes(
io::Printer* printer, const map<string, string>& variables) const {
printer->Print(variables,
"if (!this->$name$().empty()) {\n");
printer->Indent();
printer->Indent();
const FieldDescriptor* key_field =
descriptor_->message_type()->FindFieldByName("key");
const FieldDescriptor* value_field =
descriptor_->message_type()->FindFieldByName("value");
if (key_field->type() == FieldDescriptor::TYPE_STRING) {
GenerateUtf8CheckCodeForString(key_field, options_, false, variables_,
"it->first.data(), it->first.length(),\n",
printer);
const bool string_key = key_field->type() == FieldDescriptor::TYPE_STRING;
const bool string_value = value_field->type() == FieldDescriptor::TYPE_STRING;
printer->Print(variables,
"typedef ::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_pointer\n"
" ConstPtr;\n");
if (string_key) {
printer->Print(variables,
"typedef ConstPtr SortItem;\n"
"typedef ::google::protobuf::internal::"
"CompareByDerefFirst<SortItem> Less;\n");
} else {
printer->Print(variables,
"typedef ::google::protobuf::internal::SortItem< $key_cpp$, ConstPtr > "
"SortItem;\n"
"typedef ::google::protobuf::internal::CompareByFirstField<SortItem> Less;\n");
}
if (value_field->type() == FieldDescriptor::TYPE_STRING) {
GenerateUtf8CheckCodeForString(value_field, options_, false, variables_,
"it->second.data(), it->second.length(),\n",
printer);
string utf8_check;
if (string_key || string_value) {
printer->Print(
"struct Utf8Check {\n"
" static void Check(ConstPtr p) {\n");
printer->Indent();
printer->Indent();
if (string_key) {
GenerateUtf8CheckCodeForString(key_field, options_, false, variables,
"p->first.data(), p->first.length(),\n",
printer);
}
if (string_value) {
GenerateUtf8CheckCodeForString(value_field, options_, false, variables,
"p->second.data(), p->second.length(),\n",
printer);
}
printer->Outdent();
printer->Outdent();
printer->Print(
" }\n"
"};\n");
utf8_check = "Utf8Check::Check";
}
printer->Outdent();
printer->Print(variables,
"\n"
"if ($deterministic$ &&\n"
" this->$name$().size() > 1) {\n"
" ::google::protobuf::scoped_array<SortItem> items(\n"
" new SortItem[this->$name$().size()]);\n"
" typedef ::google::protobuf::Map< $key_cpp$, $val_cpp$ >::size_type size_type;\n"
" size_type n = 0;\n"
" for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n"
" it = this->$name$().begin();\n"
" it != this->$name$().end(); ++it, ++n) {\n"
" items[n] = SortItem(&*it);\n"
" }\n"
" ::std::sort(&items[0], &items[n], Less());\n");
printer->Indent();
GenerateSerializationLoop(printer, variables, SupportsArenas(descriptor_),
utf8_check, "for (size_type i = 0; i < n; i++)",
string_key ? "items[i]" : "items[i].second", false);
printer->Outdent();
printer->Print(
" }\n");
// If entry is allocated by arena, its desctructor should be avoided.
if (SupportsArenas(descriptor_)) {
printer->Print(variables_,
" if (entry.get() != NULL && entry->GetArena() != NULL) {\n"
" entry.release();\n"
" }\n");
}
"} else {\n");
printer->Indent();
GenerateSerializationLoop(
printer, variables, SupportsArenas(descriptor_), utf8_check,
"for (::google::protobuf::Map< $key_cpp$, $val_cpp$ >::const_iterator\n"
" it = this->$name$().begin();\n"
" it != this->$name$().end(); ++it)",
"it", true);
printer->Outdent();
printer->Print("}\n");
printer->Outdent();
printer->Print("}\n");
}

4
src/google/protobuf/compiler/cpp/cpp_map_field.h
View File

@ -61,6 +61,10 @@ class MapFieldGenerator : public FieldGenerator {
void GenerateByteSize(io::Printer* printer) const;
private:
// A helper for GenerateSerializeWithCachedSizes{,ToArray}.
void GenerateSerializeWithCachedSizes(
io::Printer* printer, const map<string, string>& variables) const;
const FieldDescriptor* descriptor_;
const bool dependent_field_;
map<string, string> variables_;

8
src/google/protobuf/compiler/cpp/cpp_message.cc
View File

@ -2715,7 +2715,9 @@ GenerateMergeFrom(io::Printer* printer) {
"void $classname$::MergeFrom(const ::google::protobuf::Message& from) {\n"
"// @@protoc_insertion_point(generalized_merge_from_start:"
"$full_name$)\n"
" if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);\n",
" if (GOOGLE_PREDICT_FALSE(&from == this)) {\n"
" ::google::protobuf::internal::MergeFromFail(__FILE__, __LINE__);\n"
" }\n",
"classname", classname_, "full_name", descriptor_->full_name());
printer->Indent();
@ -2756,7 +2758,9 @@ GenerateMergeFrom(io::Printer* printer) {
"void $classname$::MergeFrom(const $classname$& from) {\n"
"// @@protoc_insertion_point(class_specific_merge_from_start:"
"$full_name$)\n"
" if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__);\n",
" if (GOOGLE_PREDICT_FALSE(&from == this)) {\n"
" ::google::protobuf::internal::MergeFromFail(__FILE__, __LINE__);\n"
" }\n",
"classname", classname_, "full_name", descriptor_->full_name());
printer->Indent();

18
src/google/protobuf/compiler/java/java_file.cc
View File

@ -266,9 +266,7 @@ void FileGenerator::Generate(io::Printer* printer) {
printer->Print(
"public static void registerAllExtensions(\n"
" com.google.protobuf.ExtensionRegistry$lite$ registry) {\n",
"lite",
HasDescriptorMethods(file_, context_->EnforceLite()) ? "" : "Lite");
" com.google.protobuf.ExtensionRegistryLite registry) {\n");
printer->Indent();
@ -283,6 +281,20 @@ void FileGenerator::Generate(io::Printer* printer) {
printer->Outdent();
printer->Print(
"}\n");
if (HasDescriptorMethods(file_, context_->EnforceLite())) {
// Overload registerAllExtensions for the non-lite usage to
// redundantly maintain the original signature (this is
// redundant because ExtensionRegistryLite now invokes
// ExtensionRegistry in the non-lite usage). Intent is
// to remove this in the future.
printer->Print(
"\n"
"public static void registerAllExtensions(\n"
" com.google.protobuf.ExtensionRegistry registry) {\n"
" registerAllExtensions(\n"
" (com.google.protobuf.ExtensionRegistryLite) registry);\n"
"}\n");
}
// -----------------------------------------------------------------

2
src/google/protobuf/compiler/java/java_generator.cc
View File

@ -79,8 +79,6 @@ bool JavaGenerator::Generate(const FileDescriptor* file,
file_options.generate_mutable_code = true;
} else if (options[i].first == "shared") {
file_options.generate_shared_code = true;
} else if (options[i].first == "lite") {
file_options.enforce_lite = true;
} else if (options[i].first == "annotate_code") {
file_options.annotate_code = true;
} else if (options[i].first == "annotation_list_file") {

24
src/google/protobuf/compiler/js/js_generator.cc
View File

@ -2031,9 +2031,8 @@ void Generator::GenerateClassFieldToObject(const GeneratorOptions& options,
"getter", JSGetterName(options, field, BYTES_B64));
} else {
if (field->has_default_value()) {
printer->Print("jspb.Message.getField(msg, $index$) == null ? "
"$defaultValue$ : ",
"index", JSFieldIndex(field),
printer->Print("!msg.has$name$() ? $defaultValue$ : ",
"name", JSGetterName(options, field),
"defaultValue", JSFieldDefault(field));
}
if (field->cpp_type() == FieldDescriptor::CPPTYPE_FLOAT ||
@ -2408,9 +2407,8 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
"default", Proto3PrimitiveFieldDefault(field));
} else {
if (field->has_default_value()) {
printer->Print("jspb.Message.getField(this, $index$) == null ? "
"$defaultValue$ : ",
"index", JSFieldIndex(field),
printer->Print("!this.has$name$() ? $defaultValue$ : ",
"name", JSGetterName(options, field),
"defaultValue", JSFieldDefault(field));
}
if (field->cpp_type() == FieldDescriptor::CPPTYPE_FLOAT ||
@ -2515,6 +2513,20 @@ void Generator::GenerateClassField(const GeneratorOptions& options,
"\n",
"clearedvalue", (field->is_repeated() ? "[]" : "undefined"),
"returnvalue", JSReturnClause(field));
printer->Print(
"/**\n"
" * Returns whether this field is set.\n"
" * @return{!boolean}\n"
" */\n"
"$class$.prototype.has$name$ = function() {\n"
" return jspb.Message.getField(this, $index$) != null;\n"
"};\n"
"\n"
"\n",
"class", GetPath(options, field->containing_type()),
"name", JSGetterName(options, field),
"index", JSFieldIndex(field));
}
}
}

58
src/google/protobuf/compiler/python/python_generator.cc
View File

@ -44,6 +44,7 @@
// performance-minded Python code leverage the fast C++ implementation
// directly.
#include <algorithm>
#include <google/protobuf/stubs/hash.h>
#include <limits>
#include <map>
@ -107,20 +108,25 @@ string ModuleAlias(const string& filename) {
return module_name;
}
// Keywords reserved by the Python language.
const char* const kKeywords[] = {
"False", "None", "True", "and", "as", "assert", "break",
"class", "continue", "def", "del", "elif", "else", "except",
"finally", "for", "from", "global", "if", "import", "in",
"is", "lambda", "nonlocal", "not", "or", "pass", "raise",
"return", "try", "while", "with", "yield",
};
const char* const* kKeywordsEnd =
kKeywords + (sizeof(kKeywords) / sizeof(kKeywords[0]));
// Returns an import statement of form "from X.Y.Z import T" for the given
// .proto filename.
string ModuleImportStatement(const string& filename) {
string module_name = ModuleName(filename);
int last_dot_pos = module_name.rfind('.');
if (last_dot_pos == string::npos) {
// NOTE(petya): this is not tested as it would require a protocol buffer
// outside of any package, and I don't think that is easily achievable.
return "import " + module_name;
} else {
return "from " + module_name.substr(0, last_dot_pos) + " import " +
module_name.substr(last_dot_pos + 1);
bool ContainsPythonKeyword(const string& module_name) {
vector<string> tokens = Split(module_name, ".");
for (int i = 0; i < tokens.size(); ++i) {
if (std::find(kKeywords, kKeywordsEnd, tokens[i]) != kKeywordsEnd) {
return true;
}
}
return false;
}
@ -359,10 +365,32 @@ bool Generator::Generate(const FileDescriptor* file,
void Generator::PrintImports() const {
for (int i = 0; i < file_->dependency_count(); ++i) {
const string& filename = file_->dependency(i)->name();
string import_statement = ModuleImportStatement(filename);
string module_name = ModuleName(filename);
string module_alias = ModuleAlias(filename);
printer_->Print("$statement$ as $alias$\n", "statement",
import_statement, "alias", module_alias);
if (ContainsPythonKeyword(module_name)) {
// If the module path contains a Python keyword, we have to quote the
// module name and import it using importlib. Otherwise the usual kind of
// import statement would result in a syntax error from the presence of
// the keyword.
printer_->Print("import importlib\n");
printer_->Print("$alias$ = importlib.import_module('$name$')\n", "alias",
module_alias, "name", module_name);
} else {
int last_dot_pos = module_name.rfind('.');
string import_statement;
if (last_dot_pos == string::npos) {
// NOTE(petya): this is not tested as it would require a protocol buffer
// outside of any package, and I don't think that is easily achievable.
import_statement = "import " + module_name;
} else {
import_statement = "from " + module_name.substr(0, last_dot_pos) +
" import " + module_name.substr(last_dot_pos + 1);
}
printer_->Print("$statement$ as $alias$\n", "statement", import_statement,
"alias", module_alias);
}
CopyPublicDependenciesAliases(module_alias, file_->dependency(i));
}
printer_->Print("\n");

48
src/google/protobuf/compiler/python/python_plugin_unittest.cc
View File

@ -46,6 +46,7 @@
#include <google/protobuf/testing/file.h>
#include <google/protobuf/testing/file.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/testing/googletest.h>
#include <gtest/gtest.h>
@ -115,6 +116,53 @@ TEST(PythonPluginTest, PluginTest) {
EXPECT_EQ(0, cli.Run(5, argv));
}
// This test verifies that the generated Python output uses regular imports (as
// opposed to importlib) in the usual case where the .proto file paths do not
// not contain any Python keywords.
TEST(PythonPluginTest, ImportTest) {
// Create files test1.proto and test2.proto with the former importing the
// latter.
GOOGLE_CHECK_OK(File::SetContents(TestTempDir() + "/test1.proto",
"syntax = \"proto3\";\n"
"package foo;\n"
"import \"test2.proto\";"
"message Message1 {\n"
" Message2 message_2 = 1;\n"
"}\n",
true));
GOOGLE_CHECK_OK(File::SetContents(TestTempDir() + "/test2.proto",
"syntax = \"proto3\";\n"
"package foo;\n"
"message Message2 {}\n",
true));
google::protobuf::compiler::CommandLineInterface cli;
cli.SetInputsAreProtoPathRelative(true);
python::Generator python_generator;
cli.RegisterGenerator("--python_out", &python_generator, "");
string proto_path = "-I" + TestTempDir();
string python_out = "--python_out=" + TestTempDir();
const char* argv[] = {"protoc", proto_path.c_str(), "-I.", python_out.c_str(),
"test1.proto"};
ASSERT_EQ(0, cli.Run(5, argv));
// Loop over the lines of the generated code and verify that we find an
// ordinary Python import but do not find the string "importlib".
string output;
GOOGLE_CHECK_OK(File::GetContents(TestTempDir() + "/test1_pb2.py", &output,
true));
std::vector<string> lines = Split(output, "\n");
string expected_import = "import test2_pb2";
bool found_expected_import = false;
for (int i = 0; i < lines.size(); ++i) {
if (lines[i].find(expected_import) != string::npos) {
found_expected_import = true;
}
EXPECT_EQ(string::npos, lines[i].find("importlib"));
}
EXPECT_TRUE(found_expected_import);
}
} // namespace
} // namespace python
} // namespace compiler

1
src/google/protobuf/descriptor.proto
View File

@ -45,6 +45,7 @@ option java_package = "com.google.protobuf";
option java_outer_classname = "DescriptorProtos";
option csharp_namespace = "Google.Protobuf.Reflection";
option objc_class_prefix = "GPB";
option java_generate_equals_and_hash = true;
// descriptor.proto must be optimized for speed because reflection-based
// algorithms don't work during bootstrapping.

2
src/google/protobuf/extension_set_heavy.cc
View File

@ -341,7 +341,7 @@ bool ExtensionSet::ParseMessageSet(io::CodedInputStream* input,
int ExtensionSet::SpaceUsedExcludingSelf() const {
int total_size =
extensions_.size() * sizeof(map<int, Extension>::value_type);
extensions_.size() * sizeof(ExtensionMap::value_type);
for (ExtensionMap::const_iterator iter = extensions_.begin(),
end = extensions_.end();
iter != end;

6
src/google/protobuf/generated_message_util.cc
View File

@ -73,6 +73,12 @@ int StringSpaceUsedExcludingSelf(const string& str) {
void MergeFromFail(const char* file, int line) {
GOOGLE_CHECK(false) << file << ":" << line;
// Open-source GOOGLE_CHECK(false) is not NORETURN.
exit(1);
}
} // namespace internal
} // namespace protobuf
} // namespace google

6
src/google/protobuf/generated_message_util.h
View File

@ -41,8 +41,8 @@
#include <assert.h>
#include <string>
#include <google/protobuf/stubs/once.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/once.h>
namespace google {
@ -115,6 +115,10 @@ class ArenaString;
ArenaString* ReadArenaString(::google::protobuf::io::CodedInputStream* input,
::google::protobuf::Arena* arena);
// Helper function to crash on merge failure.
// Moved out of generated code to reduce binary size.
void MergeFromFail(const char* file, int line) GOOGLE_ATTRIBUTE_NORETURN;
} // namespace internal
} // namespace protobuf

8
src/google/protobuf/io/coded_stream.cc
View File

@ -649,13 +649,16 @@ bool CodedInputStream::Refresh() {
// CodedOutputStream =================================================
bool CodedOutputStream::default_serialization_deterministic_ = false;
CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output)
: output_(output),
buffer_(NULL),
buffer_size_(0),
total_bytes_(0),
had_error_(false),
aliasing_enabled_(false) {
aliasing_enabled_(false),
serialization_deterministic_is_overridden_(false) {
// Eagerly Refresh() so buffer space is immediately available.
Refresh();
// The Refresh() may have failed. If the client doesn't write any data,
@ -671,7 +674,8 @@ CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output,
buffer_size_(0),
total_bytes_(0),
had_error_(false),
aliasing_enabled_(false) {
aliasing_enabled_(false),
serialization_deterministic_is_overridden_(false) {
if (do_eager_refresh) {
// Eagerly Refresh() so buffer space is immediately available.
Refresh();

47
src/google/protobuf/io/coded_stream.h
View File

@ -813,6 +813,44 @@ class LIBPROTOBUF_EXPORT CodedOutputStream {
// created.
bool HadError() const { return had_error_; }
// Deterministic serialization, if requested, guarantees that for a given
// binary, equal messages will always be serialized to the same bytes. This
// implies:
// . repeated serialization of a message will return the same bytes
// . different processes of the same binary (which may be executing on
// different machines) will serialize equal messages to the same bytes.
//
// Note the deterministic serialization is NOT canonical across languages; it
// is also unstable across different builds with schema changes due to unknown
// fields. Users who need canonical serialization, e.g., persistent storage in
// a canonical form, fingerprinting, etc., should define their own
// canonicalization specification and implement the serializer using
// reflection APIs rather than relying on this API.
//
// If determinisitc serialization is requested, the serializer will
// sort map entries by keys in lexicographical order or numerical order.
// (This is an implementation detail and may subject to change.)
//
// There are two ways to determine whether serialization should be
// deterministic for this CodedOutputStream. If SetSerializationDeterministic
// has not yet been called, then the default comes from the global default,
// which is false, until SetDefaultSerializationDeterministic has been called.
// Otherwise, SetSerializationDeterministic has been called, and the last
// value passed to it is all that matters.
void SetSerializationDeterministic(bool value) {
serialization_deterministic_is_overridden_ = true;
serialization_deterministic_override_ = value;
}
// See above. Also, note that users of this CodedOutputStream may need to
// call IsSerializationDeterminstic() to serialize in the intended way. This
// CodedOutputStream cannot enforce a desire for deterministic serialization
// by itself.
bool IsSerializationDeterminstic() const {
return serialization_deterministic_is_overridden_ ?
serialization_deterministic_override_ :
default_serialization_deterministic_;
}
private:
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CodedOutputStream);
@ -822,6 +860,10 @@ class LIBPROTOBUF_EXPORT CodedOutputStream {
int total_bytes_; // Sum of sizes of all buffers seen so far.
bool had_error_; // Whether an error occurred during output.
bool aliasing_enabled_; // See EnableAliasing().
// See SetSerializationDeterministic() regarding these three fields.
bool serialization_deterministic_is_overridden_;
bool serialization_deterministic_override_;
static bool default_serialization_deterministic_;
// Advance the buffer by a given number of bytes.
void Advance(int amount);
@ -849,6 +891,11 @@ class LIBPROTOBUF_EXPORT CodedOutputStream {
uint64 value, uint8* target);
static int VarintSize32Fallback(uint32 value);
// See above. Other projects may use "friend" to allow them to call this.
static void SetDefaultSerializationDeterministic() {
default_serialization_deterministic_ = true;
}
};
// inline methods ====================================================

26
src/google/protobuf/map_entry_lite.h
View File

@ -535,6 +535,32 @@ class MapEntryLite : public MessageLite {
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MapEntryLite);
};
// Helpers for deterministic serialization =============================
// This struct can be used with any generic sorting algorithm. If the Key
// type is relatively small and easy to copy then copying Keys into an
// array of SortItems can be beneficial. Then all the data the sorting
// algorithm needs to touch is in that one array.
template <typename Key, typename PtrToKeyValuePair> struct SortItem {
SortItem() {}
explicit SortItem(PtrToKeyValuePair p) : first(p->first), second(p) {}
Key first;
PtrToKeyValuePair second;
};
template <typename T> struct CompareByFirstField {
bool operator()(const T& a, const T& b) const {
return a.first < b.first;
}
};
template <typename T> struct CompareByDerefFirst {
bool operator()(const T& a, const T& b) const {
return a->first < b->first;
}
};
} // namespace internal
} // namespace protobuf

20
src/google/protobuf/map_proto2_unittest.proto
View File

@ -64,3 +64,23 @@ message TestEnumMapPlusExtra {
message TestImportEnumMap {
map<int32, protobuf_unittest_import.ImportEnumForMap> import_enum_amp = 1;
}
message TestIntIntMap {
map<int32, int32> m = 1;
}
// Test all key types: string, plus the non-floating-point scalars.
message TestMaps {
map<int32, TestIntIntMap> m_int32 = 1;
map<int64, TestIntIntMap> m_int64 = 2;
map<uint32, TestIntIntMap> m_uint32 = 3;
map<uint64, TestIntIntMap> m_uint64 = 4;
map<sint32, TestIntIntMap> m_sint32 = 5;
map<sint64, TestIntIntMap> m_sint64 = 6;
map<fixed32, TestIntIntMap> m_fixed32 = 7;
map<fixed64, TestIntIntMap> m_fixed64 = 8;
map<sfixed32, TestIntIntMap> m_sfixed32 = 9;
map<sfixed64, TestIntIntMap> m_sfixed64 = 10;
map<bool, TestIntIntMap> m_bool = 11;
map<string, TestIntIntMap> m_string = 12;
}

77
src/google/protobuf/map_test.cc
View File

@ -74,6 +74,7 @@
#include <google/protobuf/io/tokenizer.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
#include <google/protobuf/util/time_util.h>
#include <google/protobuf/util/message_differencer.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/stubs/substitute.h>
#include <gmock/gmock.h>
@ -2869,6 +2870,82 @@ TEST(WireFormatForMapFieldTest, MapParseHelpers) {
}
}
// Deterministic Serialization Test ==========================================
template <typename T>
static string DeterministicSerialization(const T& t) {
const int size = t.ByteSize();
string result(size, '\0');
io::ArrayOutputStream array_stream(string_as_array(&result), size);
io::CodedOutputStream output_stream(&array_stream);
output_stream.SetSerializationDeterministic(true);
t.SerializeWithCachedSizes(&output_stream);
EXPECT_FALSE(output_stream.HadError());
EXPECT_EQ(size, output_stream.ByteCount());
return result;
}
// Helper to test the serialization of the first arg against a golden file.
static void TestDeterministicSerialization(const protobuf_unittest::TestMaps& t,
const string& filename) {
string expected;
GOOGLE_CHECK_OK(File::GetContents(
TestSourceDir() + "/google/protobuf/testdata/" + filename,
&expected, true));
const string actual = DeterministicSerialization(t);
EXPECT_EQ(expected, actual);
protobuf_unittest::TestMaps u;
EXPECT_TRUE(u.ParseFromString(actual));
EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(u, t));
}
// Helper for MapSerializationTest. Return a 7-bit ASCII string.
static string ConstructKey(uint64 n) {
string s(n % static_cast<uint64>(9), '\0');
if (s.empty()) {
return StrCat(n);
} else {
while (n != 0) {
s[n % s.size()] = (n >> 10) & 0x7f;
n /= 888;
}
return s;
}
}
TEST(MapSerializationTest, Deterministic) {
const int kIters = 25;
protobuf_unittest::TestMaps t;
protobuf_unittest::TestIntIntMap inner;
(*inner.mutable_m())[0] = (*inner.mutable_m())[10] =
(*inner.mutable_m())[-200] = 0;
uint64 frog = 9;
const uint64 multiplier = 0xa29cd16f;
for (int i = 0; i < kIters; i++) {
const int32 i32 = static_cast<int32>(frog & 0xffffffff);
const uint32 u32 = static_cast<uint32>(i32) * 91919;
const int64 i64 = static_cast<int64>(frog);
const uint64 u64 = frog * static_cast<uint64>(187321);
const bool b = i32 > 0;
const string s = ConstructKey(frog);
(*inner.mutable_m())[i] = i32;
(*t.mutable_m_int32())[i32] = (*t.mutable_m_sint32())[i32] =
(*t.mutable_m_sfixed32())[i32] = inner;
(*t.mutable_m_uint32())[u32] = (*t.mutable_m_fixed32())[u32] = inner;
(*t.mutable_m_int64())[i64] = (*t.mutable_m_sint64())[i64] =
(*t.mutable_m_sfixed64())[i64] = inner;
(*t.mutable_m_uint64())[u64] = (*t.mutable_m_fixed64())[u64] = inner;
(*t.mutable_m_bool())[b] = inner;
(*t.mutable_m_string())[s] = inner;
(*t.mutable_m_string())[s + string(1 << (u32 % static_cast<uint32>(9)),
b)] = inner;
inner.mutable_m()->erase(i);
frog = frog * multiplier + i;
frog ^= (frog >> 41);
}
TestDeterministicSerialization(t, "golden_message_maps");
}
// Text Format Test =================================================
TEST(TextFormatMapTest, SerializeAndParse) {

18
src/google/protobuf/map_type_handler.h
View File

@ -166,10 +166,10 @@ class MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type> {
io::CodedOutputStream* output);
static inline uint8* InternalWriteToArray(int field,
const MapEntryAccessorType& value,
bool deterministic, uint8* output);
bool deterministic, uint8* target);
static inline uint8* WriteToArray(int field,
const MapEntryAccessorType& value,
uint8* output);
uint8* target);
// Functions to manipulate data on memory. ========================
static inline const Type& GetExternalReference(const Type* value);
@ -227,11 +227,11 @@ class MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type> {
int field, \
const MapEntryAccessorType& value, \
bool deterministic, \
uint8* output); \
uint8* target); \
static inline uint8* WriteToArray(int field, \
const MapEntryAccessorType& value, \
uint8* output) { \
return InternalWriteToArray(field, value, false, output); \
uint8* target) { \
return InternalWriteToArray(field, value, false, target); \
} \
static inline const MapEntryAccessorType& GetExternalReference( \
const TypeOnMemory& value); \
@ -374,9 +374,9 @@ template <typename Type>
inline uint8*
MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type>::InternalWriteToArray(
int field, const MapEntryAccessorType& value, bool deterministic,
uint8* output) {
uint8* target) {
return WireFormatLite::InternalWriteMessageToArray(field, value,
deterministic, output);
deterministic, target);
}
#define WRITE_METHOD(FieldType, DeclaredType) \
@ -390,8 +390,8 @@ MapTypeHandler<WireFormatLite::TYPE_MESSAGE, Type>::InternalWriteToArray(
inline uint8* \
MapTypeHandler<WireFormatLite::TYPE_##FieldType, \
Type>::InternalWriteToArray( \
int field, const MapEntryAccessorType& value, bool, uint8* output) { \
return WireFormatLite::Write##DeclaredType##ToArray(field, value, output); \
int field, const MapEntryAccessorType& value, bool, uint8* target) { \
return WireFormatLite::Write##DeclaredType##ToArray(field, value, target); \
}
WRITE_METHOD(STRING , String)

2
src/google/protobuf/message.cc
View File

@ -62,8 +62,6 @@ namespace protobuf {
using internal::WireFormat;
using internal::ReflectionOps;
Message::~Message() {}
void Message::MergeFrom(const Message& from) {
const Descriptor* descriptor = GetDescriptor();
GOOGLE_CHECK_EQ(from.GetDescriptor(), descriptor)

2
src/google/protobuf/message.h
View File

@ -179,7 +179,7 @@ struct Metadata {
class LIBPROTOBUF_EXPORT Message : public MessageLite {
public:
inline Message() {}
virtual ~Message();
virtual ~Message() {}
// Basic Operations ------------------------------------------------

2
src/google/protobuf/message_lite.cc
View File

@ -46,8 +46,6 @@
namespace google {
namespace protobuf {
MessageLite::~MessageLite() {}
string MessageLite::InitializationErrorString() const {
return "(cannot determine missing fields for lite message)";
}

2
src/google/protobuf/message_lite.h
View File

@ -81,7 +81,7 @@ namespace internal {
class LIBPROTOBUF_EXPORT MessageLite {
public:
inline MessageLite() {}
virtual ~MessageLite();
virtual ~MessageLite() {}
// Basic Operations ------------------------------------------------

BIN
src/google/protobuf/testdata/golden_message_maps vendored Normal file
View File

Binary file not shown.

14
src/google/protobuf/text_format.cc
View File

@ -759,6 +759,20 @@ class TextFormat::Parser::ParserImpl {
}
return true;
}
if (TryConsume("[")) {
while (true) {
if (!LookingAt("{") && !LookingAt("<")) {
DO(SkipFieldValue());
} else {
DO(SkipFieldMessage());
}
if (TryConsume("]")) {
break;
}
DO(Consume(","));
}
return true;
}
// Possible field values other than string:
// 12345 => TYPE_INTEGER
// -12345 => TYPE_SYMBOL + TYPE_INTEGER

28
src/google/protobuf/unknown_field_set.cc
View File

@ -69,28 +69,14 @@ const UnknownFieldSet* UnknownFieldSet::default_instance() {
return default_unknown_field_set_instance_;
}
UnknownFieldSet::UnknownFieldSet()
: fields_(NULL) {}
UnknownFieldSet::~UnknownFieldSet() {
Clear();
delete fields_;
}
void UnknownFieldSet::ClearFallback() {
if (fields_ != NULL) {
for (int i = 0; i < fields_->size(); i++) {
(*fields_)[i].Delete();
}
delete fields_;
fields_ = NULL;
}
}
void UnknownFieldSet::ClearAndFreeMemory() {
if (fields_ != NULL) {
Clear();
}
GOOGLE_DCHECK(fields_ != NULL && fields_->size() > 0);
int n = fields_->size();
do {
(*fields_)[--n].Delete();
} while (n > 0);
delete fields_;
fields_ = NULL;
}
void UnknownFieldSet::InternalMergeFrom(const UnknownFieldSet& other) {

8
src/google/protobuf/unknown_field_set.h
View File

@ -241,8 +241,14 @@ class LIBPROTOBUF_EXPORT UnknownField {
// ===================================================================
// inline implementations
inline UnknownFieldSet::UnknownFieldSet() : fields_(NULL) {}
inline UnknownFieldSet::~UnknownFieldSet() { Clear(); }
inline void UnknownFieldSet::ClearAndFreeMemory() { Clear(); }
inline void UnknownFieldSet::Clear() {
if (fields_) {
if (fields_ != NULL) {
ClearFallback();
}
}

29
src/google/protobuf/util/internal/default_value_objectwriter.cc
View File

@ -64,6 +64,7 @@ DefaultValueObjectWriter::DefaultValueObjectWriter(
type_(type),
current_(NULL),
root_(NULL),
suppress_empty_list_(false),
field_scrub_callback_(NULL),
ow_(ow) {}
@ -184,12 +185,10 @@ void DefaultValueObjectWriter::RegisterFieldScrubCallBack(
field_scrub_callback_.reset(field_scrub_callback.release());
}
DefaultValueObjectWriter::Node::Node(const string& name,
const google::protobuf::Type* type,
NodeKind kind, const DataPiece& data,
bool is_placeholder,
const vector<string>& path,
FieldScrubCallBack* field_scrub_callback)
DefaultValueObjectWriter::Node::Node(
const string& name, const google::protobuf::Type* type, NodeKind kind,
const DataPiece& data, bool is_placeholder, const vector<string>& path,
bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback)
: name_(name),
type_(type),
kind_(kind),
@ -197,6 +196,7 @@ DefaultValueObjectWriter::Node::Node(const string& name,
data_(data),
is_placeholder_(is_placeholder),
path_(path),
suppress_empty_list_(suppress_empty_list),
field_scrub_callback_(field_scrub_callback) {}
DefaultValueObjectWriter::Node* DefaultValueObjectWriter::Node::FindChild(
@ -230,6 +230,9 @@ void DefaultValueObjectWriter::Node::WriteTo(ObjectWriter* ow) {
// Write out lists. If we didn't have any list in response, write out empty
// list.
if (kind_ == LIST) {
// Suppress empty lists if requested.
if (suppress_empty_list_ && is_placeholder_) return;
ow->StartList(name_);
WriteChildren(ow);
ow->EndList();
@ -366,7 +369,7 @@ void DefaultValueObjectWriter::Node::PopulateChildren(
field.json_name(), field_type, kind,
kind == PRIMITIVE ? CreateDefaultDataPieceForField(field, typeinfo)
: DataPiece::NullData(),
true, path, field_scrub_callback_));
true, path, suppress_empty_list_, field_scrub_callback_));
new_children.push_back(child.release());
}
// Adds all leftover nodes in children_ to the beginning of new_child.
@ -462,7 +465,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject(
if (current_ == NULL) {
vector<string> path;
root_.reset(new Node(name.ToString(), &type_, OBJECT, DataPiece::NullData(),
false, path, field_scrub_callback_.get()));
false, path, suppress_empty_list_,
field_scrub_callback_.get()));
root_->PopulateChildren(typeinfo_);
current_ = root_.get();
return this;
@ -478,7 +482,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject(
: NULL),
OBJECT, DataPiece::NullData(), false,
child == NULL ? current_->path() : child->path(),
field_scrub_callback_.get()));
suppress_empty_list_, field_scrub_callback_.get()));
child = node.get();
current_->AddChild(node.release());
}
@ -509,7 +513,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList(
if (current_ == NULL) {
vector<string> path;
root_.reset(new Node(name.ToString(), &type_, LIST, DataPiece::NullData(),
false, path, field_scrub_callback_.get()));
false, path, suppress_empty_list_,
field_scrub_callback_.get()));
current_ = root_.get();
return this;
}
@ -519,7 +524,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList(
google::protobuf::scoped_ptr<Node> node(
new Node(name.ToString(), NULL, LIST, DataPiece::NullData(), false,
child == NULL ? current_->path() : child->path(),
field_scrub_callback_.get()));
suppress_empty_list_, field_scrub_callback_.get()));
child = node.get();
current_->AddChild(node.release());
}
@ -577,7 +582,7 @@ void DefaultValueObjectWriter::RenderDataPiece(StringPiece name,
google::protobuf::scoped_ptr<Node> node(
new Node(name.ToString(), NULL, PRIMITIVE, data, false,
child == NULL ? current_->path() : child->path(),
field_scrub_callback_.get()));
suppress_empty_list_, field_scrub_callback_.get()));
child = node.get();
current_->AddChild(node.release());
} else {

12
src/google/protobuf/util/internal/default_value_objectwriter.h
View File

@ -122,6 +122,10 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
// field_scrub_callback pointer is also transferred to this class
void RegisterFieldScrubCallBack(FieldScrubCallBackPtr field_scrub_callback);
// If set to true, empty lists are suppressed from output when default values
// are written.
void set_suppress_empty_list(bool value) { suppress_empty_list_ = value; }
private:
enum NodeKind {
PRIMITIVE = 0,
@ -136,7 +140,7 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
public:
Node(const string& name, const google::protobuf::Type* type, NodeKind kind,
const DataPiece& data, bool is_placeholder, const vector<string>& path,
FieldScrubCallBack* field_scrub_callback);
bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback);
virtual ~Node() {
for (int i = 0; i < children_.size(); ++i) {
delete children_[i];
@ -212,6 +216,9 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
// Path of the field of this node
std::vector<string> path_;
// Whether to suppress empty list output.
bool suppress_empty_list_;
// Pointer to function for determining whether a field needs to be scrubbed
// or not. This callback is owned by the creator of this node.
FieldScrubCallBack* field_scrub_callback_;
@ -257,6 +264,9 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter {
// The stack to hold the path of Nodes from current_ to root_;
std::stack<Node*> stack_;
// Whether to suppress output of empty lists.
bool suppress_empty_list_;
// Unique Pointer to function for determining whether a field needs to be
// scrubbed or not.
FieldScrubCallBackPtr field_scrub_callback_;

33
src/google/protobuf/util/internal/default_value_objectwriter_test.cc
View File

@ -149,6 +149,39 @@ TEST_P(DefaultValueObjectWriterTest, ShouldRetainUnknownField) {
}
class DefaultValueObjectWriterSuppressListTest
: public BaseDefaultValueObjectWriterTest {
protected:
DefaultValueObjectWriterSuppressListTest()
: BaseDefaultValueObjectWriterTest(DefaultValueTest::descriptor()) {
testing_->set_suppress_empty_list(true);
}
~DefaultValueObjectWriterSuppressListTest() {}
};
INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest,
DefaultValueObjectWriterSuppressListTest,
::testing::Values(
testing::USE_TYPE_RESOLVER));
TEST_P(DefaultValueObjectWriterSuppressListTest, Empty) {
// Set expectation. Emtpy lists should be suppressed.
expects_.StartObject("")
->RenderDouble("doubleValue", 0.0)
->RenderFloat("floatValue", 0.0)
->RenderInt64("int64Value", 0)
->RenderUint64("uint64Value", 0)
->RenderInt32("int32Value", 0)
->RenderUint32("uint32Value", 0)
->RenderBool("boolValue", false)
->RenderString("stringValue", "")
->RenderBytes("bytesValue", "")
->RenderString("enumValue", "ENUM_FIRST")
->EndObject();
// Actual testing
testing_->StartObject("")->EndObject();
}
} // namespace testing
} // namespace converter
} // namespace util

34
src/google/protobuf/util/internal/proto_writer.cc
View File

@ -298,7 +298,9 @@ ProtoWriter::ProtoElement::ProtoElement(const TypeInfo* typeinfo,
proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3),
type_(type),
size_index_(-1),
array_index_(-1) {
array_index_(-1),
// oneof_indices_ values are 1-indexed (0 means not present).
oneof_indices_(type.oneofs_size() + 1) {
if (!proto3_) {
required_fields_ = GetRequiredFields(type_);
}
@ -312,13 +314,15 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent,
ow_(this->parent()->ow_),
parent_field_(field),
typeinfo_(this->parent()->typeinfo_),
proto3_(this->parent()->proto3_),
proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3),
type_(type),
size_index_(
!is_list && field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE
? ow_->size_insert_.size()
: -1),
array_index_(is_list ? 0 : -1) {
array_index_(is_list ? 0 : -1),
// oneof_indices_ values are 1-indexed (0 means not present).
oneof_indices_(type_.oneofs_size() + 1) {
if (!is_list) {
if (ow_->IsRepeated(*field)) {
// Update array_index_ if it is an explicit list.
@ -411,11 +415,11 @@ string ProtoWriter::ProtoElement::ToString() const {
}
bool ProtoWriter::ProtoElement::IsOneofIndexTaken(int32 index) {
return ContainsKey(oneof_indices_, index);
return oneof_indices_[index];
}
void ProtoWriter::ProtoElement::TakeOneofIndex(int32 index) {
InsertIfNotPresent(&oneof_indices_, index);
oneof_indices_[index] = true;
}
void ProtoWriter::InvalidName(StringPiece unknown_name, StringPiece message) {
@ -573,10 +577,19 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField(
// Pushing a ProtoElement and then pop it off at the end for 2 purposes:
// error location reporting and required field accounting.
element_.reset(new ProtoElement(element_.release(), &field, type, false));
//
// For proto3, since there is no required field tracking, we only need to push
// ProtoElement for error cases.
if (!element_->proto3()) {
element_.reset(new ProtoElement(element_.release(), &field, type, false));
}
if (field.kind() == google::protobuf::Field_Kind_TYPE_UNKNOWN ||
field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) {
// Push a ProtoElement for location reporting purposes.
if (element_->proto3()) {
element_.reset(new ProtoElement(element_.release(), &field, type, false));
}
InvalidValue(field.type_url().empty()
? google::protobuf::Field_Kind_Name(field.kind())
: field.type_url(),
@ -657,11 +670,18 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField(
}
if (!status.ok()) {
// Push a ProtoElement for location reporting purposes.
if (element_->proto3()) {
element_.reset(new ProtoElement(element_.release(), &field, type, false));
}
InvalidValue(google::protobuf::Field_Kind_Name(field.kind()),
status.error_message());
element_.reset(element()->pop());
return this;
}
element_.reset(element()->pop());
if (!element_->proto3()) element_.reset(element()->pop());
return this;
}

9
src/google/protobuf/util/internal/proto_writer.h
View File

@ -32,8 +32,8 @@
#define GOOGLE_PROTOBUF_UTIL_CONVERTER_PROTO_WRITER_H__
#include <deque>
#include <google/protobuf/stubs/hash.h>
#include <string>
#include <vector>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/io/coded_stream.h>
@ -45,6 +45,7 @@
#include <google/protobuf/util/internal/structured_objectwriter.h>
#include <google/protobuf/util/type_resolver.h>
#include <google/protobuf/stubs/bytestream.h>
#include <google/protobuf/stubs/hash.h>
namespace google {
namespace protobuf {
@ -191,6 +192,8 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter {
// generate an error.
void TakeOneofIndex(int32 index);
bool proto3() { return proto3_; }
private:
// Used for access to variables of the enclosing instance of
// ProtoWriter.
@ -203,7 +206,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter {
// TypeInfo to lookup types.
const TypeInfo* typeinfo_;
// Whether the root type is a proto3 or not.
// Whether the type_ is proto3 or not.
bool proto3_;
// Additional variables if this element is a message:
@ -221,7 +224,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter {
// Set of oneof indices already seen for the type_. Used to validate
// incoming messages so no more than one oneof is set.
hash_set<int32> oneof_indices_;
std::vector<bool> oneof_indices_;
GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoElement);
};

60
src/google/protobuf/util/json_util.cc
View File

@ -177,6 +177,66 @@ util::Status JsonToBinaryString(TypeResolver* resolver,
resolver, type_url, &input_stream, &output_stream, options);
}
namespace {
const char* kTypeUrlPrefix = "type.googleapis.com";
TypeResolver* generated_type_resolver_ = NULL;
GOOGLE_PROTOBUF_DECLARE_ONCE(generated_type_resolver_init_);
string GetTypeUrl(const Message& message) {
return string(kTypeUrlPrefix) + "/" + message.GetDescriptor()->full_name();
}
void DeleteGeneratedTypeResolver() { delete generated_type_resolver_; }
void InitGeneratedTypeResolver() {
generated_type_resolver_ = NewTypeResolverForDescriptorPool(
kTypeUrlPrefix, DescriptorPool::generated_pool());
::google::protobuf::internal::OnShutdown(&DeleteGeneratedTypeResolver);
}
TypeResolver* GetGeneratedTypeResolver() {
::google::protobuf::GoogleOnceInit(&generated_type_resolver_init_, &InitGeneratedTypeResolver);
return generated_type_resolver_;
}
} // namespace
util::Status MessageToJsonString(const Message& message, string* output,
const JsonOptions& options) {
const DescriptorPool* pool = message.GetDescriptor()->file()->pool();
TypeResolver* resolver =
pool == DescriptorPool::generated_pool()
? GetGeneratedTypeResolver()
: NewTypeResolverForDescriptorPool(kTypeUrlPrefix, pool);
util::Status result =
BinaryToJsonString(resolver, GetTypeUrl(message),
message.SerializeAsString(), output, options);
if (pool != DescriptorPool::generated_pool()) {
delete resolver;
}
return result;
}
util::Status JsonStringToMessage(const string& input, Message* message,
const JsonParseOptions& options) {
const DescriptorPool* pool = message->GetDescriptor()->file()->pool();
TypeResolver* resolver =
pool == DescriptorPool::generated_pool()
? GetGeneratedTypeResolver()
: NewTypeResolverForDescriptorPool(kTypeUrlPrefix, pool);
string binary;
util::Status result = JsonToBinaryString(
resolver, GetTypeUrl(*message), input, &binary, options);
if (result.ok() && !message->ParseFromString(binary)) {
result =
util::Status(util::error::INVALID_ARGUMENT,
"JSON transcoder produced invalid protobuf output.");
}
if (pool != DescriptorPool::generated_pool()) {
delete resolver;
}
return result;
}
} // namespace util
} // namespace protobuf
} // namespace google

25
src/google/protobuf/util/json_util.h
View File

@ -33,6 +33,7 @@
#ifndef GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__
#define GOOGLE_PROTOBUF_UTIL_JSON_UTIL_H__
#include <google/protobuf/message.h>
#include <google/protobuf/util/type_resolver.h>
#include <google/protobuf/stubs/bytestream.h>
@ -69,6 +70,30 @@ struct JsonPrintOptions {
// DEPRECATED. Use JsonPrintOptions instead.
typedef JsonPrintOptions JsonOptions;
// Converts from protobuf message to JSON. This is a simple wrapper of
// BinaryToJsonString(). It will use the DescriptorPool of the passed-in
// message to resolve Any types.
util::Status MessageToJsonString(const Message& message,
string* output,
const JsonOptions& options);
inline util::Status MessageToJsonString(const Message& message,
string* output) {
return MessageToJsonString(message, output, JsonOptions());
}
// Converts from JSON to protobuf message. This is a simple wrapper of
// JsonStringToBinary(). It will use the DescriptorPool of the passed-in
// message to resolve Any types.
util::Status JsonStringToMessage(const string& input,
Message* message,
const JsonParseOptions& options);
inline util::Status JsonStringToMessage(const string& input,
Message* message) {
return JsonStringToMessage(input, message, JsonParseOptions());
}
// Converts protobuf binary data to JSON.
// The conversion will fail if:
// 1. TypeResolver fails to resolve a type.

60
src/google/protobuf/util/json_util_test.cc
View File

@ -34,6 +34,8 @@
#include <string>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/descriptor_database.h>
#include <google/protobuf/dynamic_message.h>
#include <google/protobuf/util/json_format_proto3.pb.h>
#include <google/protobuf/util/type_resolver.h>
#include <google/protobuf/util/type_resolver_util.h>
@ -63,28 +65,21 @@ static string GetTypeUrl(const Descriptor* message) {
class JsonUtilTest : public testing::Test {
protected:
JsonUtilTest() {
resolver_.reset(NewTypeResolverForDescriptorPool(
kTypeUrlPrefix, DescriptorPool::generated_pool()));
}
string ToJson(const Message& message, const JsonPrintOptions& options) {
string result;
GOOGLE_CHECK_OK(BinaryToJsonString(resolver_.get(),
GetTypeUrl(message.GetDescriptor()),
message.SerializeAsString(), &result, options));
GOOGLE_CHECK_OK(MessageToJsonString(message, &result, options));
return result;
}
bool FromJson(const string& json, Message* message,
const JsonParseOptions& options) {
string binary;
if (!JsonToBinaryString(resolver_.get(),
GetTypeUrl(message->GetDescriptor()), json, &binary,
options)
.ok()) {
return false;
}
return message->ParseFromString(binary);
return JsonStringToMessage(json, message, options).ok();
}
bool FromJson(const string& json, Message* message) {
return FromJson(json, message, JsonParseOptions());
}
google::protobuf::scoped_ptr<TypeResolver> resolver_;
@ -189,6 +184,45 @@ TEST_F(JsonUtilTest, TestParseErrors) {
EXPECT_FALSE(FromJson("{\"int32Value\":2147483648}", &m, options));
}
TEST_F(JsonUtilTest, TestDynamicMessage) {
// Some random message but good enough to test the wrapper functions.
string input =
"{\n"
" \"int32Value\": 1024,\n"
" \"repeatedInt32Value\": [1, 2],\n"
" \"messageValue\": {\n"
" \"value\": 2048\n"
" },\n"
" \"repeatedMessageValue\": [\n"
" {\"value\": 40}, {\"value\": 96}\n"
" ]\n"
"}\n";
// Create a new DescriptorPool with the same protos as the generated one.
DescriptorPoolDatabase database(*DescriptorPool::generated_pool());
DescriptorPool pool(&database);
// A dynamic version of the test proto.
DynamicMessageFactory factory;
google::protobuf::scoped_ptr<Message> message(factory.GetPrototype(
pool.FindMessageTypeByName("proto3.TestMessage"))->New());
EXPECT_TRUE(FromJson(input, message.get()));
// Convert to generated message for easy inspection.
TestMessage generated;
EXPECT_TRUE(generated.ParseFromString(message->SerializeAsString()));
EXPECT_EQ(1024, generated.int32_value());
ASSERT_EQ(2, generated.repeated_int32_value_size());
EXPECT_EQ(1, generated.repeated_int32_value(0));
EXPECT_EQ(2, generated.repeated_int32_value(1));
EXPECT_EQ(2048, generated.message_value().value());
ASSERT_EQ(2, generated.repeated_message_value_size());
EXPECT_EQ(40, generated.repeated_message_value(0).value());
EXPECT_EQ(96, generated.repeated_message_value(1).value());
JsonOptions options;
EXPECT_EQ(ToJson(generated, options), ToJson(*message, options));
}
typedef pair<char*, int> Segment;
// A ZeroCopyOutputStream that writes to multiple buffers.
class SegmentedZeroCopyOutputStream : public io::ZeroCopyOutputStream {

6
src/google/protobuf/wire_format_lite.cc
View File

@ -466,7 +466,8 @@ void WireFormatLite::WriteGroupMaybeToArray(int field_number,
const int size = value.GetCachedSize();
uint8* target = output->GetDirectBufferForNBytesAndAdvance(size);
if (target != NULL) {
uint8* end = value.SerializeWithCachedSizesToArray(target);
uint8* end = value.InternalSerializeWithCachedSizesToArray(
output->IsSerializationDeterminstic(), target);
GOOGLE_DCHECK_EQ(end - target, size);
} else {
value.SerializeWithCachedSizes(output);
@ -482,7 +483,8 @@ void WireFormatLite::WriteMessageMaybeToArray(int field_number,
output->WriteVarint32(size);
uint8* target = output->GetDirectBufferForNBytesAndAdvance(size);
if (target != NULL) {
uint8* end = value.SerializeWithCachedSizesToArray(target);
uint8* end = value.InternalSerializeWithCachedSizesToArray(
output->IsSerializationDeterminstic(), target);
GOOGLE_DCHECK_EQ(end - target, size);
} else {
value.SerializeWithCachedSizes(output);