AuroraMiddleware/protobuf
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

Factored conformance tests so they can run in-process.

Browse Source 
This is necessary for testing on iOS.

Change-Id: I54ec1e3aa2e9fbfff9a6cd6580920a6a62069b63
This commit is contained in:
Josh Haberman 2015-04-14 13:45:39 -07:00
parent 8fe135d212
commit 4e63b52088
5 changed files with 415 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
conformance/Makefile.am
View File

@ -7,12 +7,12 @@ protoc_outputs = \
conformance.pb.cc \
conformance.pb.h
bin_PROGRAMS = conformance-test conformance-cpp
bin_PROGRAMS = conformance-test-runner conformance-cpp
conformance_test_LDADD = $(top_srcdir)/src/libprotobuf.la
conformance_test_SOURCES = conformance_test.cc
nodist_conformance_test_SOURCES = conformance.pb.cc
conformance_test_CPPFLAGS = -I$(top_srcdir)/src
conformance_test_runner_LDADD = $(top_srcdir)/src/libprotobuf.la
conformance_test_runner_SOURCES = conformance_test.cc conformance_test_runner.cc
nodist_conformance_test_runner_SOURCES = conformance.pb.cc
conformance_test_runner_CPPFLAGS = -I$(top_srcdir)/src
conformance_cpp_LDADD = $(top_srcdir)/src/libprotobuf.la
conformance_cpp_SOURCES = conformance_cpp.cc
@ -46,5 +46,5 @@ MAINTAINERCLEANFILES = \
Makefile.in
# Targets for actually running tests.
test_cpp: unittest_proto_middleman conformance-test conformance-cpp
./conformance-test ./conformance-cpp
test_cpp: unittest_proto_middleman conformance-test-runner conformance-cpp
./conformance-test-runner ./conformance-cpp

30
conformance/conformance.proto
View File

@ -32,27 +32,23 @@ syntax = "proto3";
package conformance;
// This defines the conformance testing protocol. This protocol exists between
// the conformance tester process (the "tester") and the process whose protobuf
// implemention is being tested (the "testee"). The tester forks the testee and
// communicates with it over its stdin/stdout:
// the conformance test suite itself and the code being tested. For each test,
// the suite will send a ConformanceRequest message and expect a
// ConformanceResponse message.
//
// +--------+ pipe +----------+
// | tester | <------> | testee |
// | | | |
// | C++ | | any lang |
// +--------+ +----------+
// You can either run the tests in two different ways:
//
// The tester contains all of the test cases and their expected output.
// The testee is a simple program written in the target language that reads
// each test case and attempts to produce acceptable output for it.
// 1. in-process (using the interface in conformance_test.h).
//
// Every test consists of a ConformanceRequest/ConformanceResponse
// request/reply pair. The protocol on the pipe is simply:
// 2. as a sub-process communicating over a pipe. Information about how to
// do this is in conformance_test_runner.cc.
//
// 1. tester sends 4-byte length N
// 2. tester sends N bytes representing a ConformanceRequest proto
// 3. testee sends 4-byte length M
// 4. testee sends M bytes representing a ConformanceResponse proto
// Pros/cons of the two approaches:
//
// - running as a sub-process is much simpler for languages other than C/C++.
//
// - running as a sub-process may be more tricky in unusual environments like
// iOS apps, where fork/stdin/stdout are not available.
// Represents a single test case's input. The testee should:
//

265
conformance/conformance_test.cc
View File

@ -28,13 +28,13 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <errno.h>
#include <stdarg.h>
#include <unistd.h>
#include <string>
#include "conformance.pb.h"
#include "conformance_test.h"
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/stringprintf.h>
#include <google/protobuf/wire_format_lite.h>
using conformance::ConformanceRequest;
@ -45,167 +45,7 @@ using google::protobuf::FieldDescriptor;
using google::protobuf::internal::WireFormatLite;
using std::string;
int write_fd;
int read_fd;
int successes;
int failures;
bool verbose = false;
string Escape(const string& str) {
// TODO.
return str;
}
#define STRINGIFY(x) #x
#define TOSTRING(x) STRINGIFY(x)
#define CHECK_SYSCALL(call) \
if (call < 0) { \
perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \
exit(1); \
}
// TODO(haberman): make this work on Windows, instead of using these
// UNIX-specific APIs.
//
// There is a platform-agnostic API in
// src/google/protobuf/compiler/subprocess.h
//
// However that API only supports sending a single message to the subprocess.
// We really want to be able to send messages and receive responses one at a
// time:
//
// 1. Spawning a new process for each test would take way too long for thousands
// of tests and subprocesses like java that can take 100ms or more to start
// up.
//
// 2. Sending all the tests in one big message and receiving all results in one
// big message would take away our visibility about which test(s) caused a
// crash or other fatal error. It would also give us only a single failure
// instead of all of them.
void SpawnTestProgram(char *executable) {
int toproc_pipe_fd[2];
int fromproc_pipe_fd[2];
if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) {
perror("pipe");
exit(1);
}
pid_t pid = fork();
if (pid < 0) {
perror("fork");
exit(1);
}
if (pid) {
// Parent.
CHECK_SYSCALL(close(toproc_pipe_fd[0]));
CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
write_fd = toproc_pipe_fd[1];
read_fd = fromproc_pipe_fd[0];
} else {
// Child.
CHECK_SYSCALL(close(STDIN_FILENO));
CHECK_SYSCALL(close(STDOUT_FILENO));
CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO));
CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO));
CHECK_SYSCALL(close(toproc_pipe_fd[0]));
CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
CHECK_SYSCALL(close(toproc_pipe_fd[1]));
CHECK_SYSCALL(close(fromproc_pipe_fd[0]));
char *const argv[] = {executable, NULL};
CHECK_SYSCALL(execv(executable, argv)); // Never returns.
}
}
/* Invoking of tests **********************************************************/
void ReportSuccess() {
successes++;
}
void ReportFailure(const char *fmt, ...) {
va_list args;
va_start(args, fmt);
vfprintf(stderr, fmt, args);
va_end(args);
failures++;
}
void CheckedWrite(int fd, const void *buf, size_t len) {
if (write(fd, buf, len) != len) {
GOOGLE_LOG(FATAL) << "Error writing to test program: " << strerror(errno);
}
}
void CheckedRead(int fd, void *buf, size_t len) {
size_t ofs = 0;
while (len > 0) {
ssize_t bytes_read = read(fd, (char*)buf + ofs, len);
if (bytes_read == 0) {
GOOGLE_LOG(FATAL) << "Unexpected EOF from test program";
} else if (bytes_read < 0) {
GOOGLE_LOG(FATAL) << "Error reading from test program: " << strerror(errno);
}
len -= bytes_read;
ofs += bytes_read;
}
}
void RunTest(const ConformanceRequest& request, ConformanceResponse* response) {
string serialized;
request.SerializeToString(&serialized);
uint32_t len = serialized.size();
CheckedWrite(write_fd, &len, sizeof(uint32_t));
CheckedWrite(write_fd, serialized.c_str(), serialized.size());
CheckedRead(read_fd, &len, sizeof(uint32_t));
serialized.resize(len);
CheckedRead(read_fd, (void*)serialized.c_str(), len);
if (!response->ParseFromString(serialized)) {
GOOGLE_LOG(FATAL) << "Could not parse response proto from tested process.";
}
if (verbose) {
fprintf(stderr, "conformance_test: request=%s, response=%s\n",
request.ShortDebugString().c_str(),
response->ShortDebugString().c_str());
}
}
void DoExpectParseFailureForProto(const string& proto, int line) {
ConformanceRequest request;
ConformanceResponse response;
request.set_protobuf_payload(proto);
// We don't expect output, but if the program erroneously accepts the protobuf
// we let it send its response as this. We must not leave it unspecified.
request.set_requested_output(ConformanceRequest::PROTOBUF);
RunTest(request, &response);
if (response.result_case() == ConformanceResponse::kParseError) {
ReportSuccess();
} else {
ReportFailure("Should have failed, but didn't. Line: %d, Request: %s, "
"response: %s\n",
line,
request.ShortDebugString().c_str(),
response.ShortDebugString().c_str());
}
}
// Expect that this precise protobuf will cause a parse error.
#define ExpectParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__)
// Expect that this protobuf will cause a parse error, even if it is followed
// by valid protobuf data. We can try running this twice: once with this
// data verbatim and once with this data followed by some valid data.
//
// TODO(haberman): implement the second of these.
#define ExpectHardParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__)
namespace {
/* Routines for building arbitrary protos *************************************/
@ -299,7 +139,78 @@ uint32_t GetFieldNumberForType(WireFormatLite::FieldType type, bool repeated) {
return 0;
}
void TestPrematureEOFForType(WireFormatLite::FieldType type) {
} // anonymous namespace
namespace google {
namespace protobuf {
void ConformanceTestSuite::ReportSuccess() {
successes_++;
}
void ConformanceTestSuite::ReportFailure(const char *fmt, ...) {
va_list args;
va_start(args, fmt);
StringAppendV(&output_, fmt, args);
va_end(args);
failures_++;
}
void ConformanceTestSuite::RunTest(const ConformanceRequest& request,
ConformanceResponse* response) {
string serialized_request;
string serialized_response;
request.SerializeToString(&serialized_request);
runner_->RunTest(serialized_request, &serialized_response);
if (!response->ParseFromString(serialized_response)) {
response->Clear();
response->set_runtime_error("response proto could not be parsed.");
}
if (verbose_) {
StringAppendF(&output_, "conformance test: request=%s, response=%s\n",
request.ShortDebugString().c_str(),
response->ShortDebugString().c_str());
}
}
void ConformanceTestSuite::DoExpectParseFailureForProto(const string& proto,
int line) {
ConformanceRequest request;
ConformanceResponse response;
request.set_protobuf_payload(proto);
// We don't expect output, but if the program erroneously accepts the protobuf
// we let it send its response as this. We must not leave it unspecified.
request.set_requested_output(ConformanceRequest::PROTOBUF);
RunTest(request, &response);
if (response.result_case() == ConformanceResponse::kParseError) {
ReportSuccess();
} else {
ReportFailure("Should have failed, but didn't. Line: %d, Request: %s, "
"response: %s\n",
line,
request.ShortDebugString().c_str(),
response.ShortDebugString().c_str());
}
}
// Expect that this precise protobuf will cause a parse error.
#define ExpectParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__)
// Expect that this protobuf will cause a parse error, even if it is followed
// by valid protobuf data. We can try running this twice: once with this
// data verbatim and once with this data followed by some valid data.
//
// TODO(haberman): implement the second of these.
#define ExpectHardParseFailureForProto(proto) DoExpectParseFailureForProto(proto, __LINE__)
void ConformanceTestSuite::TestPrematureEOFForType(
WireFormatLite::FieldType type) {
// Incomplete values for each wire type.
static const string incompletes[6] = {
string("\x80"), // VARINT
@ -376,20 +287,24 @@ void TestPrematureEOFForType(WireFormatLite::FieldType type) {
}
}
int main(int argc, char *argv[]) {
if (argc < 2) {
fprintf(stderr, "Usage: conformance_test <test-program>\n");
exit(1);
}
SpawnTestProgram(argv[1]);
void ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
std::string* output) {
runner_ = runner;
output_.clear();
successes_ = 0;
failures_ = 0;
for (int i = 1; i <= FieldDescriptor::MAX_TYPE; i++) {
TestPrematureEOFForType(static_cast<WireFormatLite::FieldType>(i));
}
fprintf(stderr, "conformance_test: completed %d tests for %s, %d successes, "
"%d failures.\n", successes + failures, argv[1], successes,
failures);
StringAppendF(&output_,
"CONFORMANCE SUITE FINISHED: completed %d tests, %d successes, "
"%d failures.\n",
successes_ + failures_, successes_, failures_);
output->assign(output_);
}
} // namespace protobuf
} // namespace google

109
conformance/conformance_test.h Normal file
View File

@ -0,0 +1,109 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// This file defines a protocol for running the conformance test suite
// in-process. In other words, the suite itself will run in the same process as
// the code under test.
//
// For pros and cons of this approach, please see conformance.proto.
#ifndef CONFORMANCE_CONFORMANCE_TEST_H
#define CONFORMANCE_CONFORMANCE_TEST_H
#include <string>
#include <google/protobuf/wire_format_lite.h>
namespace conformance {
class ConformanceRequest;
class ConformanceResponse;
} // namespace conformance
namespace google {
namespace protobuf {
class ConformanceTestRunner {
public:
// Call to run a single conformance test.
//
// "input" is a serialized conformance.ConformanceRequest.
// "output" should be set to a serialized conformance.ConformanceResponse.
//
// If there is any error in running the test itself, set "runtime_error" in
// the response.
virtual void RunTest(const std::string& input, std::string* output) = 0;
};
// Class representing the test suite itself. To run it, implement your own
// class derived from ConformanceTestRunner and then write code like:
//
// class MyConformanceTestRunner : public ConformanceTestRunner {
// public:
// virtual void RunTest(...) {
// // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE.
// }
// };
//
// int main() {
// MyConformanceTestRunner runner;
// google::protobuf::ConformanceTestSuite suite;
//
// std::string output;
// suite.RunSuite(&runner, &output);
// }
//
class ConformanceTestSuite {
public:
ConformanceTestSuite() : verbose_(false) {}
// Run all the conformance tests against the given test runner.
// Test output will be stored in "output".
void RunSuite(ConformanceTestRunner* runner, std::string* output);
private:
void ReportSuccess();
void ReportFailure(const char* fmt, ...);
void RunTest(const conformance::ConformanceRequest& request,
conformance::ConformanceResponse* response);
void DoExpectParseFailureForProto(const std::string& proto, int line);
void TestPrematureEOFForType(
google::protobuf::internal::WireFormatLite::FieldType type);
ConformanceTestRunner* runner_;
int successes_;
int failures_;
bool verbose_;
std::string output_;
};
} // namespace protobuf
} // namespace google
#endif // CONFORMANCE_CONFORMANCE_TEST_H

196
conformance/conformance_test_runner.cc Normal file
View File

@ -0,0 +1,196 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// This file contains a program for running the test suite in a separate
// process. The other alternative is to run the suite in-process. See
// conformance.proto for pros/cons of these two options.
//
// This program will fork the process under test and communicate with it over
// its stdin/stdout:
//
// +--------+ pipe +----------+
// | tester | <------> | testee |
// | | | |
// | C++ | | any lang |
// +--------+ +----------+
//
// The tester contains all of the test cases and their expected output.
// The testee is a simple program written in the target language that reads
// each test case and attempts to produce acceptable output for it.
//
// Every test consists of a ConformanceRequest/ConformanceResponse
// request/reply pair. The protocol on the pipe is simply:
//
// 1. tester sends 4-byte length N
// 2. tester sends N bytes representing a ConformanceRequest proto
// 3. testee sends 4-byte length M
// 4. testee sends M bytes representing a ConformanceResponse proto
#include <errno.h>
#include <unistd.h>
#include "conformance.pb.h"
#include "conformance_test.h"
using conformance::ConformanceRequest;
using conformance::ConformanceResponse;
using google::protobuf::internal::scoped_array;
#define STRINGIFY(x) #x
#define TOSTRING(x) STRINGIFY(x)
#define CHECK_SYSCALL(call) \
if (call < 0) { \
perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \
exit(1); \
}
// Test runner that spawns the process being tested and communicates with it
// over a pipe.
class ForkPipeRunner : public google::protobuf::ConformanceTestRunner {
public:
ForkPipeRunner(const std::string &executable)
: executable_(executable), running_(false) {}
void RunTest(const std::string& request, std::string* response) {
if (!running_) {
SpawnTestProgram();
}
uint32_t len = request.size();
CheckedWrite(write_fd_, &len, sizeof(uint32_t));
CheckedWrite(write_fd_, request.c_str(), request.size());
CheckedRead(read_fd_, &len, sizeof(uint32_t));
response->resize(len);
CheckedRead(read_fd_, (void*)response->c_str(), len);
}
private:
// TODO(haberman): make this work on Windows, instead of using these
// UNIX-specific APIs.
//
// There is a platform-agnostic API in
// src/google/protobuf/compiler/subprocess.h
//
// However that API only supports sending a single message to the subprocess.
// We really want to be able to send messages and receive responses one at a
// time:
//
// 1. Spawning a new process for each test would take way too long for thousands
// of tests and subprocesses like java that can take 100ms or more to start
// up.
//
// 2. Sending all the tests in one big message and receiving all results in one
// big message would take away our visibility about which test(s) caused a
// crash or other fatal error. It would also give us only a single failure
// instead of all of them.
void SpawnTestProgram() {
int toproc_pipe_fd[2];
int fromproc_pipe_fd[2];
if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) {
perror("pipe");
exit(1);
}
pid_t pid = fork();
if (pid < 0) {
perror("fork");
exit(1);
}
if (pid) {
// Parent.
CHECK_SYSCALL(close(toproc_pipe_fd[0]));
CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
write_fd_ = toproc_pipe_fd[1];
read_fd_ = fromproc_pipe_fd[0];
running_ = true;
} else {
// Child.
CHECK_SYSCALL(close(STDIN_FILENO));
CHECK_SYSCALL(close(STDOUT_FILENO));
CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO));
CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO));
CHECK_SYSCALL(close(toproc_pipe_fd[0]));
CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
CHECK_SYSCALL(close(toproc_pipe_fd[1]));
CHECK_SYSCALL(close(fromproc_pipe_fd[0]));
scoped_array<char> executable(new char[executable_.size()]);
memcpy(executable.get(), executable_.c_str(), executable_.size());
char *const argv[] = {executable.get(), NULL};
CHECK_SYSCALL(execv(executable.get(), argv)); // Never returns.
}
}
void CheckedWrite(int fd, const void *buf, size_t len) {
if (write(fd, buf, len) != len) {
GOOGLE_LOG(FATAL) << "Error writing to test program: " << strerror(errno);
}
}
void CheckedRead(int fd, void *buf, size_t len) {
size_t ofs = 0;
while (len > 0) {
ssize_t bytes_read = read(fd, (char*)buf + ofs, len);
if (bytes_read == 0) {
GOOGLE_LOG(FATAL) << "Unexpected EOF from test program";
} else if (bytes_read < 0) {
GOOGLE_LOG(FATAL) << "Error reading from test program: " << strerror(errno);
}
len -= bytes_read;
ofs += bytes_read;
}
}
int write_fd_;
int read_fd_;
bool running_;
std::string executable_;
};
int main(int argc, char *argv[]) {
if (argc < 2) {
fprintf(stderr, "Usage: conformance-test-runner <test-program>\n");
exit(1);
}
ForkPipeRunner runner(argv[1]);
google::protobuf::ConformanceTestSuite suite;
std::string output;
suite.RunSuite(&runner, &output);
fwrite(output.c_str(), 1, output.size(), stderr);
}