Merge pull request #1295 from haberman/docker

Add support for running on Jenkins

.travis.yml

@@ -10,7 +10,7 @@ os:
 # The Objective C build needs Xcode 7.0 or later.
 osx_image: xcode7.2
 script:
-  - ./travis.sh $CONFIG
+  - ./tests.sh $CONFIG
 env:
   - CONFIG=cpp
   - CONFIG=cpp_distcheck

jenkins/README.md

@@ -0,0 +1,6 @@
+
+Jenkins Infrastructure
+----------------------
+
+The scripts in this directory serve as plumbing for running the protobuf
+tests under Jenkins.

jenkins/build_and_run_docker.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Builds docker image and runs a command under it.
+# This is a generic script that is configured with the following variables:
+#
+# DOCKERFILE_DIR - Directory in which Dockerfile file is located.
+# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root)
+# OUTPUT_DIR - Directory that will be copied from inside docker after finishing.
+# $@ - Extra args to pass to docker run
+
+
+set -ex
+
+cd $(dirname $0)/..
+git_root=$(pwd)
+cd -
+
+# Use image name based on Dockerfile location checksum
+DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ )
+
+# Make sure docker image has been built. Should be instantaneous if so.
+docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR
+
+# Ensure existence of ccache directory
+CCACHE_DIR=/tmp/protobuf-ccache
+mkdir -p $CCACHE_DIR
+
+# Choose random name for docker container
+CONTAINER_NAME="build_and_run_docker_$(uuidgen)"
+
+# Run command inside docker
+docker run \
+  "$@" \
+  -e CCACHE_DIR=$CCACHE_DIR \
+  -e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \
+  -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
+  -v "$git_root:/var/local/jenkins/protobuf:ro" \
+  -v $CCACHE_DIR:$CCACHE_DIR \
+  -w /var/local/git/protobuf \
+  --name=$CONTAINER_NAME \
+  $DOCKER_IMAGE_NAME \
+  bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true"
+
+# Copy output artifacts
+if [ "$OUTPUT_DIR" != "" ]
+then
+  docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true"
+fi
+
+# remove the container, possibly killing it first
+docker rm -f $CONTAINER_NAME || true
+
+if [ "$FAILED" != "" ]
+then
+  exit 1
+fi

jenkins/buildcmds/README.md

@@ -0,0 +1,6 @@
+
+Jenkins Build Commands
+----------------------
+
+The scripts in this directory are designed to be top-level entry points for
+Jenkins projects.

jenkins/buildcmds/pull_request.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Jenkins as the entry point for
+# running the "pull request" project:
+#
+#   https://grpc-testing.appspot.com/view/Protocol%20Buffers/job/protobuf_pull_request/
+#
+# This script selects a specific Dockerfile (for building a Docker image) and
+# a script to run inside that image.  Then we delegate to the general
+# build_and_run_docker.sh script.
+
+export DOCKERFILE_DIR=jenkins/docker
+export DOCKER_RUN_SCRIPT=jenkins/pull_request_in_docker.sh
+export OUTPUT_DIR=testoutput
+./jenkins/build_and_run_docker.sh

jenkins/docker/Dockerfile

@@ -0,0 +1,130 @@
+# This Dockerfile specifies the recipe for creating an image for the tests
+# to run in.
+#
+# We install as many test dependencies here as we can, because these setup
+# steps can be cached.  They do *not* run every time we run the build.
+# The Docker image is only rebuilt when the Dockerfile (ie. this file)
+# changes.
+
+# Base Dockerfile for gRPC dev images
+FROM debian:latest
+
+# Apt source for old Python versions.
+RUN echo 'deb http://ppa.launchpad.net/fkrull/deadsnakes/ubuntu trusty main' > /etc/apt/sources.list.d/deadsnakes.list && \
+  apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DB82666C
+
+# Apt source for Oracle Java.
+run echo 'deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main' > /etc/apt/sources.list.d/webupd8team-java-trusty.list && \
+  apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \
+  echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
+
+# Apt source for Mono
+run echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list && \
+  echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list && \
+  apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
+
+# Install dependencies.  We start with the basic ones require to build protoc
+# and the C++ build
+RUN apt-get update && apt-get install -y \
+  autoconf \
+  autotools-dev \
+  build-essential \
+  bzip2 \
+  ccache \
+  curl \
+  gcc \
+  git \
+  libc6 \
+  libc6-dbg \
+  libc6-dev \
+  libgtest-dev \
+  libtool \
+  make \
+  parallel \
+  time \
+  wget \
+  # -- For csharp --
+  mono-devel \
+  referenceassemblies-pcl \
+  nunit \
+  # -- For all Java builds -- \
+  maven \
+  # -- For java_jdk6 -- \
+  #   oops! not in jessie. too old? openjdk-6-jdk \
+  # -- For java_jdk7 -- \
+  openjdk-7-jdk \
+  # -- For java_oracle7 -- \
+  oracle-java7-installer \
+  # -- For python / python_cpp -- \
+  python-setuptools \
+  python-pip \
+  python-dev \
+  python2.6-dev \
+  python3.3-dev \
+  python3.4-dev \
+  # -- For Ruby --
+  ruby \
+  && apt-get clean
+
+##################
+# C# dependencies
+
+RUN wget www.nuget.org/NuGet.exe -O /usr/local/bin/nuget.exe
+
+##################
+# Python dependencies
+
+# These packages exist in apt-get, but their versions are too old, so we have
+# to get updates from pip.
+
+RUN pip install pip --upgrade
+RUN pip install virtualenv tox yattag
+
+
+##################
+# Ruby dependencies
+
+# Install rvm
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+RUN \curl -sSL https://get.rvm.io | bash -s stable
+
+# Install Ruby 2.1
+RUN /bin/bash -l -c "rvm install ruby-2.1"
+RUN /bin/bash -l -c "rvm use --default ruby-2.1"
+RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
+RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"
+RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc"
+RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
+
+##################
+# Java dependencies
+
+# This step requires compiling protoc. :(
+
+ENV MAVEN_REPO /var/maven_local_repository
+ENV MVN mvn --batch-mode
+
+RUN cd /tmp && \
+  git clone https://github.com/google/protobuf.git && \
+  cd protobuf && \
+  ./autogen.sh && \
+  ./configure && \
+  make -j6 && \
+  cd java && \
+  $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO -P lite && \
+  $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO && \
+  cd ../javanano && \
+  $MVN install dependency:go-offline -Dmaven.repo.local=$MAVEN_REPO
+
+##################
+# Prepare ccache
+
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
+# Define the default command.
+CMD ["bash"]

jenkins/make_test_output.py

@@ -0,0 +1,91 @@
+"""Gathers output from test runs and create an XML file in JUnit format.
+
+The output files from the individual tests have been written in a directory
+structure like:
+
+  $DIR/joblog  (output from "parallel --joblog joblog")
+  $DIR/logs/1/cpp/stdout
+  $DIR/logs/1/cpp/stderr
+  $DIR/logs/1/csharp/stdout
+  $DIR/logs/1/csharp/stderr
+  $DIR/logs/1/java_jdk7/stdout
+  $DIR/logs/1/java_jdk7/stderr
+  etc.
+
+This script bundles them into a single output XML file so Jenkins can show
+detailed test results.  It runs as the last step before the Jenkins build
+finishes.
+"""
+
+import os;
+import sys;
+from yattag import Doc
+from collections import defaultdict
+
+def readtests(basedir):
+  tests = defaultdict(dict)
+
+  # Sample input (note: separators are tabs).
+  #
+  # Seq	Host	Starttime	Runtime	Send	Receive	Exitval	Signal	Command
+  # 1	:	1456263838.313	0.005	0	0	0	0	echo A
+  with open(basedir + "/joblog") as jobs:
+    firstline = next(jobs)
+    for line in jobs:
+      values = line.split("\t")
+
+      name = values[8].split()[-1]
+      test = tests[name]
+      test["name"] = name
+      test["time"] = values[3]
+
+      exitval = values[6]
+      if int(exitval):
+        # We don't have a more specific message.  User should look at stderr.
+        test["failure"] = "TEST FAILURE"
+      else:
+        test["failure"] = False
+
+  for testname in os.listdir(basedir + "/logs/1"):
+    test = tests[testname]
+
+    with open(basedir + "/logs/1/" + testname + "/stdout") as f:
+      test["stdout"] = f.read()
+
+    with open(basedir + "/logs/1/" + testname + "/stderr") as f:
+      test["stderr"] = f.read()
+
+  # The cpp test is special since it doesn't run under parallel so doesn't show
+  # up in the job log.
+  tests["cpp"]["name"] = "cpp"
+
+  with open(basedir + '/logs/1/cpp/build_time', 'r') as f:
+    tests["cpp"]["time"] = f.read().strip()
+  tests["cpp"]["failure"] = False
+
+  ret = tests.values()
+  ret.sort(key=lambda x: x["name"])
+
+  return ret
+
+def genxml(tests):
+  doc, tag, text = Doc().tagtext()
+
+  with tag("testsuites"):
+    with tag("testsuite", name="Protobuf Tests"):
+      for test in tests:
+        with tag("testcase", name=test["name"], classname=test["name"],
+                             time=test["time"]):
+          with tag("system-out"):
+            text(test["stdout"])
+          with tag("system-err"):
+            text(test["stderr"])
+          if test["failure"]:
+            with tag("failure"):
+              text(test["failure"])
+
+  return doc.getvalue()
+
+sys.stderr.write("make_test_output.py: writing XML from directory: " +
+                 sys.argv[1] + "\n");
+print genxml(readtests(sys.argv[1]))

jenkins/pull_request_in_docker.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# This is the script that runs inside Docker, once the image has been built,
+# to execute all tests for the "pull request" project.
+
+WORKSPACE_BASE=`pwd`
+MY_DIR="$(dirname "$0")"
+TEST_SCRIPT=$MY_DIR/../tests.sh
+BUILD_DIR=/tmp/protobuf
+
+set -e  # exit immediately on error
+set -x  # display all commands
+
+# The protobuf repository is mounted into our Docker image, but read-only.
+# We clone into a directory inside Docker (this is faster than cp).
+rm -rf $BUILD_DIR
+mkdir -p $BUILD_DIR
+cd $BUILD_DIR
+git clone /var/local/jenkins/protobuf
+cd protobuf
+
+# Set up the directory where our test output is going to go.
+OUTPUT_DIR=`mktemp -d`
+LOG_OUTPUT_DIR=$OUTPUT_DIR/logs
+mkdir -p $LOG_OUTPUT_DIR/1/cpp
+
+################################################################################
+# cpp build needs to run first, non-parallelized, so that protoc is available
+# for other builds.
+
+# Output filenames to follow the overall scheme used by parallel, ie:
+#  $DIR/logs/1/cpp/stdout
+#  $DIR/logs/1/cpp/stderr
+#  $DIR/logs/1/csharp/stdout
+#  $DIR/logs/1/csharp/stderr
+#  $DIR/logs/1/java_jdk7/stdout
+#  $DIR/logs/1/java_jdk7/stderr
+CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout
+CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr
+
+# Time the C++ build, so we can put this info in the test output.
+# It's important that we get /usr/bin/time (which supports -f and -o) and not
+# the bash builtin "time" which doesn't.
+TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time"
+
+$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2)
+
+# Other tests are run in parallel.
+
+parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \
+  csharp \
+  java_jdk7 \
+  javanano_jdk7 \
+  java_oracle7 \
+  javanano_oracle7 \
+  python \
+  python_cpp \
+  ruby21 \
+  || true  # Process test results even if tests fail.
+
+cat $OUTPUT_DIR/joblog
+
+# The directory that is copied from Docker back into the Jenkins workspace.
+COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput
+mkdir -p $COPY_FROM_DOCKER
+TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/testresults.xml
+
+# Process all the output files from "parallel" and package them into a single
+# .xml file with detailed, broken-down test output.
+python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
+
+ls -l $TESTOUTPUT_XML_FILE

tests.sh

@@ -1,17 +1,24 @@
-#!/usr/bin/env bash
+#!/bin/bash
+#
+# Build and runs tests for the protobuf project.  The tests as written here are
+# used by both Jenkins and Travis, though some specialized logic is required to
+# handle the differences between them.
 
-# Note: travis currently does not support testing more than one language so the
-# .travis.yml cheats and claims to only be cpp.  If they add multiple language
-# support, this should probably get updated to install steps and/or
-# rvm/gemfile/jdk/etc. entries rather than manually doing the work.
-
-# .travis.yml uses matrix.exclude to block the cases where app-get can't be
-# use to install things.
+on_travis() {
+  if [ "$TRAVIS" == "true" ]; then
+    "$@"
+  fi
+}
 
 # For when some other test needs the C++ main build, including protoc and
 # libprotobuf.
 internal_build_cpp() {
-  if [ $(uname -s) == "Linux" ]; then
+  if [ -f src/protoc ]; then
+    # Already built.
+    return
+  fi
+
+  if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then
     # Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more
     # decent C++ 11 support in order to compile conformance tests.
     sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
@@ -42,7 +49,9 @@ build_csharp() {
   # need to really build protoc, but it's simplest to keep with the
   # conventions of the other builds.
   internal_build_cpp
+  NUGET=/usr/local/bin/nuget.exe
 
+  if [ "$TRAVIS" == "true" ]; then
     # Install latest version of Mono
     sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
     echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
@@ -50,8 +59,10 @@ build_csharp() {
     sudo apt-get update -qq
     sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit
     wget www.nuget.org/NuGet.exe -O nuget.exe
+    NUGET=../../nuget.exe
+  fi
 
-  (cd csharp/src; mono ../../nuget.exe restore)
+  (cd csharp/src; mono $NUGET restore)
   csharp/buildall.sh
   cd conformance && make test_csharp && cd ..
 }
@@ -78,40 +89,54 @@ use_java() {
   version=$1
   case "$version" in
     jdk6)
-      sudo apt-get install openjdk-6-jdk
+      on_travis sudo apt-get install openjdk-6-jdk
       export PATH=/usr/lib/jvm/java-6-openjdk-amd64/bin:$PATH
       ;;
     jdk7)
-      sudo apt-get install openjdk-7-jdk
+      on_travis sudo apt-get install openjdk-7-jdk
       export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
       ;;
     oracle7)
+      if [ "$TRAVIS" == "true" ]; then
         sudo apt-get install python-software-properties # for apt-add-repository
         echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \
           sudo debconf-set-selections
         yes | sudo apt-add-repository ppa:webupd8team/java
         yes | sudo apt-get install oracle-java7-installer
+      fi;
       export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH
       ;;
   esac
 
+  if [ "$TRAVIS" != "true" ]; then
+    MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository
+    MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY"
+  fi;
+
   which java
   java -version
 }
 
+# --batch-mode supresses download progress output that spams the logs.
+MVN="mvn --batch-mode"
+
 build_java() {
+  version=$1
+  dir=java_$version
   # Java build needs `protoc`.
   internal_build_cpp
-  cd java && mvn test && mvn install
-  cd util && mvn test
+  cp -r java $dir
+  cd $dir && $MVN clean && $MVN test
   cd ../..
 }
 
+# The conformance tests are hard-coded to work with the $ROOT/java directory.
+# So this can't run in parallel with two different sets of tests.
 build_java_with_conformance_tests() {
   # Java build needs `protoc`.
   internal_build_cpp
-  cd java && mvn test && mvn install
-  cd util && mvn test && mvn assembly:single
+  cd java && $MVN test && $MVN install
+  cd util && $MVN package assembly:single
   cd ../..
   cd conformance && make test_java && cd ..
 }
@@ -119,12 +144,12 @@ build_java_with_conformance_tests() {
 build_javanano() {
   # Java build needs `protoc`.
   internal_build_cpp
-  cd javanano && mvn test && cd ..
+  cd javanano && $MVN test && cd ..
 }
 
 build_java_jdk6() {
   use_java jdk6
-  build_java
+  build_java jdk6
 }
 build_java_jdk7() {
   use_java jdk7
@@ -132,7 +157,7 @@ build_java_jdk7() {
 }
 build_java_oracle7() {
   use_java oracle7
-  build_java
+  build_java oracle7
 }
 
 build_javanano_jdk6() {
@@ -149,6 +174,9 @@ build_javanano_oracle7() {
 }
 
 internal_install_python_deps() {
+  if [ "$TRAVIS" != "true" ]; then
+    return;
+  fi
   # Install tox (OS X doesn't have pip).
   if [ $(uname -s) == "Darwin" ]; then
     sudo easy_install tox
@@ -278,6 +306,14 @@ build_javascript() {
   cd js && npm install && npm test && cd ..
 }
 
+# Note: travis currently does not support testing more than one language so the
+# .travis.yml cheats and claims to only be cpp.  If they add multiple language
+# support, this should probably get updated to install steps and/or
+# rvm/gemfile/jdk/etc. entries rather than manually doing the work.
+
+# .travis.yml uses matrix.exclude to block the cases where app-get can't be
+# use to install things.
+
 # -------- main --------
 
 if [ "$#" -ne 1 ]; then
@@ -294,10 +330,10 @@ Usage: $0 { cpp |
             objectivec_osx |
             python |
             python_cpp |
-            ruby_19 |
-            ruby_20 |
-            ruby_21 |
-            ruby_22 |
+            ruby19 |
+            ruby20 |
+            ruby21 |
+            ruby22 |
             jruby }
 "
   exit 1