| // Protocol Buffers - Google's data interchange format |
| // Copyright 2008 Google Inc. All rights reserved. |
| // https://developers.google.com/protocol-buffers/ |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are |
| // met: |
| // |
| // * Redistributions of source code must retain the above copyright |
| // notice, this list of conditions and the following disclaimer. |
| // * Redistributions in binary form must reproduce the above |
| // copyright notice, this list of conditions and the following disclaimer |
| // in the documentation and/or other materials provided with the |
| // distribution. |
| // * Neither the name of Google Inc. nor the names of its |
| // contributors may be used to endorse or promote products derived from |
| // this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| // This file contains a program for running the test suite in a separate |
| // process. The other alternative is to run the suite in-process. See |
| // conformance.proto for pros/cons of these two options. |
| // |
| // This program will fork the process under test and communicate with it over |
| // its stdin/stdout: |
| // |
| // +--------+ pipe +----------+ |
| // | tester | <------> | testee | |
| // | | | | |
| // | C++ | | any lang | |
| // +--------+ +----------+ |
| // |
| // The tester contains all of the test cases and their expected output. |
| // The testee is a simple program written in the target language that reads |
| // each test case and attempts to produce acceptable output for it. |
| // |
| // Every test consists of a ConformanceRequest/ConformanceResponse |
| // request/reply pair. The protocol on the pipe is simply: |
| // |
| // 1. tester sends 4-byte length N (little endian) |
| // 2. tester sends N bytes representing a ConformanceRequest proto |
| // 3. testee sends 4-byte length M (little endian) |
| // 4. testee sends M bytes representing a ConformanceResponse proto |
| |
| #include <algorithm> |
| #include <errno.h> |
| #include <fstream> |
| #include <sys/types.h> |
| #include <sys/wait.h> |
| #include <unistd.h> |
| #include <vector> |
| |
| #include <google/protobuf/stubs/stringprintf.h> |
| |
| #include "conformance.pb.h" |
| #include "conformance_test.h" |
| |
| using conformance::ConformanceRequest; |
| using conformance::ConformanceResponse; |
| using google::protobuf::StringAppendF; |
| using std::string; |
| using std::vector; |
| |
| #define STRINGIFY(x) #x |
| #define TOSTRING(x) STRINGIFY(x) |
| #define CHECK_SYSCALL(call) \ |
| if (call < 0) { \ |
| perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \ |
| exit(1); \ |
| } |
| |
| // Test runner that spawns the process being tested and communicates with it |
| // over a pipe. |
| class ForkPipeRunner : public google::protobuf::ConformanceTestRunner { |
| public: |
| ForkPipeRunner(const std::string &executable) |
| : child_pid_(-1), executable_(executable) {} |
| |
| virtual ~ForkPipeRunner() {} |
| |
| void RunTest(const std::string& test_name, |
| const std::string& request, |
| std::string* response) { |
| if (child_pid_ < 0) { |
| SpawnTestProgram(); |
| } |
| |
| current_test_name_ = test_name; |
| |
| uint32_t len = request.size(); |
| CheckedWrite(write_fd_, &len, sizeof(uint32_t)); |
| CheckedWrite(write_fd_, request.c_str(), request.size()); |
| |
| if (!TryRead(read_fd_, &len, sizeof(uint32_t))) { |
| // We failed to read from the child, assume a crash and try to reap. |
| GOOGLE_LOG(INFO) << "Trying to reap child, pid=" << child_pid_; |
| |
| int status; |
| waitpid(child_pid_, &status, WEXITED); |
| |
| string error_msg; |
| if (WIFEXITED(status)) { |
| StringAppendF(&error_msg, |
| "child exited, status=%d", WEXITSTATUS(status)); |
| } else if (WIFSIGNALED(status)) { |
| StringAppendF(&error_msg, |
| "child killed by signal %d", WTERMSIG(status)); |
| } |
| GOOGLE_LOG(INFO) << error_msg; |
| child_pid_ = -1; |
| |
| conformance::ConformanceResponse response_obj; |
| response_obj.set_runtime_error(error_msg); |
| response_obj.SerializeToString(response); |
| return; |
| } |
| |
| response->resize(len); |
| CheckedRead(read_fd_, (void*)response->c_str(), len); |
| } |
| |
| private: |
| // TODO(haberman): make this work on Windows, instead of using these |
| // UNIX-specific APIs. |
| // |
| // There is a platform-agnostic API in |
| // src/google/protobuf/compiler/subprocess.h |
| // |
| // However that API only supports sending a single message to the subprocess. |
| // We really want to be able to send messages and receive responses one at a |
| // time: |
| // |
| // 1. Spawning a new process for each test would take way too long for thousands |
| // of tests and subprocesses like java that can take 100ms or more to start |
| // up. |
| // |
| // 2. Sending all the tests in one big message and receiving all results in one |
| // big message would take away our visibility about which test(s) caused a |
| // crash or other fatal error. It would also give us only a single failure |
| // instead of all of them. |
| void SpawnTestProgram() { |
| int toproc_pipe_fd[2]; |
| int fromproc_pipe_fd[2]; |
| if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) { |
| perror("pipe"); |
| exit(1); |
| } |
| |
| pid_t pid = fork(); |
| if (pid < 0) { |
| perror("fork"); |
| exit(1); |
| } |
| |
| if (pid) { |
| // Parent. |
| CHECK_SYSCALL(close(toproc_pipe_fd[0])); |
| CHECK_SYSCALL(close(fromproc_pipe_fd[1])); |
| write_fd_ = toproc_pipe_fd[1]; |
| read_fd_ = fromproc_pipe_fd[0]; |
| child_pid_ = pid; |
| } else { |
| // Child. |
| CHECK_SYSCALL(close(STDIN_FILENO)); |
| CHECK_SYSCALL(close(STDOUT_FILENO)); |
| CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO)); |
| CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO)); |
| |
| CHECK_SYSCALL(close(toproc_pipe_fd[0])); |
| CHECK_SYSCALL(close(fromproc_pipe_fd[1])); |
| CHECK_SYSCALL(close(toproc_pipe_fd[1])); |
| CHECK_SYSCALL(close(fromproc_pipe_fd[0])); |
| |
| std::unique_ptr<char[]> executable(new char[executable_.size() + 1]); |
| memcpy(executable.get(), executable_.c_str(), executable_.size()); |
| executable[executable_.size()] = '\0'; |
| |
| char *const argv[] = {executable.get(), NULL}; |
| CHECK_SYSCALL(execv(executable.get(), argv)); // Never returns. |
| } |
| } |
| |
| void CheckedWrite(int fd, const void *buf, size_t len) { |
| if (write(fd, buf, len) != len) { |
| GOOGLE_LOG(FATAL) << current_test_name_ |
| << ": error writing to test program: " |
| << strerror(errno); |
| } |
| } |
| |
| bool TryRead(int fd, void *buf, size_t len) { |
| size_t ofs = 0; |
| while (len > 0) { |
| ssize_t bytes_read = read(fd, (char*)buf + ofs, len); |
| |
| if (bytes_read == 0) { |
| GOOGLE_LOG(ERROR) << current_test_name_ |
| << ": unexpected EOF from test program"; |
| return false; |
| } else if (bytes_read < 0) { |
| GOOGLE_LOG(ERROR) << current_test_name_ |
| << ": error reading from test program: " |
| << strerror(errno); |
| return false; |
| } |
| |
| len -= bytes_read; |
| ofs += bytes_read; |
| } |
| |
| return true; |
| } |
| |
| void CheckedRead(int fd, void *buf, size_t len) { |
| if (!TryRead(fd, buf, len)) { |
| GOOGLE_LOG(FATAL) << current_test_name_ |
| << ": error reading from test program: " |
| << strerror(errno); |
| } |
| } |
| |
| int write_fd_; |
| int read_fd_; |
| pid_t child_pid_; |
| std::string executable_; |
| std::string current_test_name_; |
| }; |
| |
| void UsageError() { |
| fprintf(stderr, |
| "Usage: conformance-test-runner [options] <test-program>\n"); |
| fprintf(stderr, "\n"); |
| fprintf(stderr, "Options:\n"); |
| fprintf(stderr, |
| " --failure_list <filename> Use to specify list of tests\n"); |
| fprintf(stderr, |
| " that are expected to fail. File\n"); |
| fprintf(stderr, |
| " should contain one test name per\n"); |
| fprintf(stderr, |
| " line. Use '#' for comments.\n"); |
| fprintf(stderr, |
| " --enforce_recommended Enforce that recommended test\n"); |
| fprintf(stderr, |
| " cases are also passing. Specify\n"); |
| fprintf(stderr, |
| " this flag if you want to be\n"); |
| fprintf(stderr, |
| " strictly conforming to protobuf\n"); |
| fprintf(stderr, |
| " spec.\n"); |
| exit(1); |
| } |
| |
| void ParseFailureList(const char *filename, std::vector<string>* failure_list) { |
| std::ifstream infile(filename); |
| |
| if (!infile.is_open()) { |
| fprintf(stderr, "Couldn't open failure list file: %s\n", filename); |
| exit(1); |
| } |
| |
| for (string line; getline(infile, line);) { |
| // Remove whitespace. |
| line.erase(std::remove_if(line.begin(), line.end(), ::isspace), |
| line.end()); |
| |
| // Remove comments. |
| line = line.substr(0, line.find("#")); |
| |
| if (!line.empty()) { |
| failure_list->push_back(line); |
| } |
| } |
| } |
| |
| int main(int argc, char *argv[]) { |
| char *program; |
| google::protobuf::ConformanceTestSuite suite; |
| |
| string failure_list_filename; |
| std::vector<string> failure_list; |
| |
| for (int arg = 1; arg < argc; ++arg) { |
| if (strcmp(argv[arg], "--failure_list") == 0) { |
| if (++arg == argc) UsageError(); |
| failure_list_filename = argv[arg]; |
| ParseFailureList(argv[arg], &failure_list); |
| } else if (strcmp(argv[arg], "--verbose") == 0) { |
| suite.SetVerbose(true); |
| } else if (strcmp(argv[arg], "--enforce_recommended") == 0) { |
| suite.SetEnforceRecommended(true); |
| } else if (argv[arg][0] == '-') { |
| fprintf(stderr, "Unknown option: %s\n", argv[arg]); |
| UsageError(); |
| } else { |
| if (arg != argc - 1) { |
| fprintf(stderr, "Too many arguments.\n"); |
| UsageError(); |
| } |
| program = argv[arg]; |
| } |
| } |
| |
| suite.SetFailureList(failure_list_filename, failure_list); |
| ForkPipeRunner runner(program); |
| |
| std::string output; |
| bool ok = suite.RunSuite(&runner, &output); |
| |
| fwrite(output.c_str(), 1, output.size(), stderr); |
| |
| return ok ? EXIT_SUCCESS : EXIT_FAILURE; |
| } |
