Add an integration test that setup the jenkins cluster using Docker
Right now this test just verify that some jobs answer 200 OK but we
can add more stuff later. Note that each new test can adds a second
or more (let say if we actually trigger a build) but launching the
jenkins cluster takes at least 45seconds on my laptop, so it is worth
putting more stuff in one tests before sharding it.
Change-Id: Ia3a6b0dbad3821e36fa8c0d49a1ebfb2b9343c05
diff --git a/WORKSPACE b/WORKSPACE
index 69402bf..16fa68e 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -25,6 +25,10 @@
)
docker_repositories()
+# For testing with docker
+load("//jenkins/test:docker_repository.bzl", "docker_repository")
+docker_repository()
+
# Docker base images
load("//base:docker_pull.bzl", "docker_pull")
diff --git a/jenkins/BUILD b/jenkins/BUILD
index 1f100a1..264b3af 100644
--- a/jenkins/BUILD
+++ b/jenkins/BUILD
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+package(default_visibility = ["//jenkins:__subpackages__"])
+
# Configuration for our Jenkins instance
load("@io_bazel_rules_docker//docker:docker.bzl", "docker_build")
load("//jenkins/build_defs:jenkins.bzl", "jenkins_node", "jenkins_docker_build", "jenkins_job", "jenkins_nodes", "jenkins_node_names")
@@ -352,12 +354,7 @@
srcs = glob(["config/**"]),
)
-sh_binary(
+alias(
name = "test",
- srcs = ["test-runner.sh"],
- data = [
- ":deploy.docker",
- ":jenkins-test",
- ":ubuntu-docker.docker",
- ],
+ actual = "//jenkins/test",
)
diff --git a/jenkins/test-runner.sh b/jenkins/test-runner.sh
deleted file mode 100755
index c24708d..0000000
--- a/jenkins/test-runner.sh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2016 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Runs the jenkins test images inside the docker container.
-# TODO(dmarting): should certainly centralize everything around docker testing
-# in the docker rules and replace that.
-
-# So that the docker incremental loader knows where the runfiles are.
-export PYTHON_RUNFILES="${PYTHON_RUNFILES:-${BASH_SOURCE[0]}.runfiles}"
-cd "${PYTHON_RUNFILES}"
-PYTHON_RUNFILES="${PWD}"
-
-# Port to serve Jenkins on, default is 8080.
-PORT="8080"
-
-# Machine on which the docker daemon is running, default is localhost.
-: ${DOCKER_SERVER:=localhost}
-
-VOLUMES=()
-while getopts ":p:h:s:" opt; do
- case "${opt}" in
- p)
- PORT="${OPTARG}"
- ;;
- h)
- VOLUMES=(-v "${OPTARG}:/volumes/jenkins_home")
- ;;
- s)
- VOLUMES=(-v "${OPTARG}:/volumes/secrets")
- ;;
- *)
- echo "Usage: $0 [-p <port>] [-s </volumes/secrets>] [-h </volumes/jenkins_home>]" >&2
- exit 1
- ;;
- esac
-done
-
-# Load all images.
-./io_bazel_ci/jenkins/ubuntu-docker.docker
-./io_bazel_ci/jenkins/deploy.docker
-./io_bazel_ci/jenkins/jenkins-test
-
-# Run main container, serving jenkins on port provided by the first argument,
-# defaulting to 8080.
-docker rm -f jenkins &> /dev/null # Remove latent jenkins instance.
-docker run -d \
- --env JENKINS_SERVER="http://jenkins:${PORT}" \
- --name jenkins \
- "${VOLUMES[@]}" \
- -p "${PORT}:8080" \
- -p 50000:50000 \
- bazel/jenkins:jenkins-test >/dev/null
-
-test_http() {
- return [[ "$(curl -sI "$1")" =~ ^HTTP/[0-9.]+\ 2 ]]
-}
-
-# Wait for jenkins to start-up for at most 3 minutes
-echo -n "*** Waiting for jenkins server to be up and running"
-timeout=180
-ts="$(date +%s)"
-while ! [[ "$(curl -sI "http://$DOCKER_SERVER:${PORT}/jnlpJars/slave.jar")" =~ ^HTTP/[0-9.]+\ 2 ]]
-do
- echo -n "."
- sleep 1
- if (( "$(date +%s)" - "$ts" > "$timeout" )); then
- echo
- echo "Failed to connect to Jenkins, aborting..." >&2
- exit 1
- fi
-done
-echo " ok."
-
-# Run the executor nodes, in priviledged mode for Bazel.
-container1="$(docker run -d --privileged=true \
- --link jenkins:jenkins --env JENKINS_SERVER=http://jenkins:${PORT} \
- bazel/jenkins:ubuntu-docker.docker)"
-container2="$(docker run -d --privileged=true \
- --link jenkins:jenkins --env JENKINS_SERVER=http://jenkins:${PORT} \
- bazel/jenkins:deploy.docker)"
-
-# Connect to the master container, until the user quit.
-docker attach jenkins
-
-# Kill the executor nodes and remove containers.
-docker rm -f "${container1}" > /dev/null
-docker rm -f "${container2}" > /dev/null
-docker rm -f jenkins > /dev/null
diff --git a/jenkins/test/BUILD b/jenkins/test/BUILD
new file mode 100644
index 0000000..c83df10
--- /dev/null
+++ b/jenkins/test/BUILD
@@ -0,0 +1,56 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//jenkins:__subpackages__"])
+
+sh_library(
+ name = "common",
+ srcs = ["common.sh"],
+ data = [
+ "//jenkins:deploy.docker",
+ "//jenkins:jenkins-test",
+ "//jenkins:ubuntu-docker.docker",
+ ],
+)
+
+sh_binary(
+ name = "test",
+ srcs = ["test.sh"],
+ data = [
+ "//jenkins:deploy.docker",
+ "//jenkins:jenkins-test",
+ "//jenkins:ubuntu-docker.docker",
+ ],
+ deps = [":common"],
+)
+
+sh_library(
+ name = "test-support",
+ srcs = ["test-support.sh"],
+ deps = [":common"],
+)
+
+sh_test(
+ name = "test_url_answer",
+ size = "large",
+ srcs = ["test_url_answer.sh"],
+ # TODO(dmarting): why the sh_library data does not get shipped in the sh_test data?
+ data = [
+ "//jenkins:deploy.docker",
+ "//jenkins:jenkins-test",
+ "//jenkins:ubuntu-docker.docker",
+ "@docker//:docker",
+ ],
+ deps = [":test-support"],
+)
diff --git a/jenkins/test/common.sh b/jenkins/test/common.sh
new file mode 100755
index 0000000..d82f091
--- /dev/null
+++ b/jenkins/test/common.sh
@@ -0,0 +1,107 @@
+#!/bin/bash
+#
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runs the jenkins test images inside the docker container.
+# TODO(dmarting): should certainly centralize everything around docker testing
+# in the docker rules and replace that.
+
+: ${DOCKER:=docker}
+
+load_images() {
+ # Load all images.
+ "${PYTHON_RUNFILES}/io_bazel_ci/jenkins/ubuntu-docker.docker"
+ "${PYTHON_RUNFILES}/io_bazel_ci/jenkins/deploy.docker"
+ "${PYTHON_RUNFILES}/io_bazel_ci/jenkins/jenkins-test"
+}
+
+run_jenkins_master() {
+ # Run main container, serving jenkins on port provided by the first argument,
+ # defaulting to 8080.
+ local server="${1:-127.0.0.1}"
+ local port="${2:-8080}"
+ shift 2
+ ${DOCKER} run -d \
+ --env JENKINS_SERVER="http://jenkins:${port}" \
+ -p "${server}:${port}:8080" \
+ "$@" \
+ bazel/jenkins:jenkins-test
+}
+
+get_jenkins_port() {
+ # Return the port in which the jenkins master has been mapped to
+ ${DOCKER} port "$1" 8080 | cut -d ":" -f 2
+}
+
+test_http() {
+ return [[ "$(curl -sI "$1")" =~ ^HTTP/[0-9.]+\ 2 ]]
+}
+
+# Wait for jenkins to start-up for at most 3 minutes
+wait_for_server() {
+ local server="${1:-localhost}"
+ local port="${2:-8080}"
+ local timeout=180
+ local ts="$(date +%s)"
+ echo -n "*** Waiting for jenkins server to be up and running on port ${port}"
+ while ! [[ "$(curl -sI "http://${server}:${port}/jnlpJars/slave.jar")" =~ ^HTTP/[0-9.]+\ 2 ]]
+ do
+ echo -n "."
+ sleep 1
+ if (( "$(date +%s)" - "$ts" > "$timeout" )); then
+ echo
+ echo "Failed to connect to Jenkins, aborting..." >&2
+ exit 1
+ fi
+ done
+ echo " ok."
+}
+
+run_containers() {
+ # Run the executor nodes, in priviledged mode for Bazel.
+ local jenkins="$1"
+ ${DOCKER} run -d --privileged=true \
+ --link "${jenkins}:jenkins" --env "JENKINS_SERVER=http://jenkins:8080" \
+ bazel/jenkins:ubuntu-docker.docker
+ ${DOCKER} run -d --privileged=true \
+ --link "${jenkins}:jenkins" --env "JENKINS_SERVER=http://jenkins:8080" \
+ bazel/jenkins:deploy.docker
+}
+
+kill_containers() {
+ # Kill containers and remove them
+ for i in "$@"; do
+ ${DOCKER} rm -f "$i" &>/dev/null || true
+ done
+}
+
+test_setup() {
+ # Setup the test and returns the list of started-up containers, the master first
+ local server="${1:-localhost}"
+ local port="${2:-0}"
+ shift 2
+ load_images >&2
+ local jenkins="$(run_jenkins_master "${server}" "${port}" "$@")"
+ port=$(get_jenkins_port "${jenkins}")
+ wait_for_server "${server}" "${port}" >&2
+ local containers="$(run_containers "${jenkins}" | xargs)"
+ echo "$jenkins $containers"
+}
+
+attach() {
+ # Attach to the first containers passed in argument
+ ${DOCKER} attach "$1"
+}
+
diff --git a/jenkins/test/docker_repository.bzl b/jenkins/test/docker_repository.bzl
new file mode 100644
index 0000000..4f96170
--- /dev/null
+++ b/jenkins/test/docker_repository.bzl
@@ -0,0 +1,57 @@
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Rule for importing the docker binary for tests (experimental)."""
+
+def _impl(ctx):
+ docker = ctx.which("docker")
+ if docker == None:
+ # We cannot find docker, we won't be able to run tests depending
+ # on it, silently ignoring.
+ ctx.file("BUILD",
+ "\n".join([
+ "filegroup(",
+ " name = 'docker',",
+ " visibility = ['//visibility:public'],",
+ ")"
+ ]))
+ else:
+ exports = []
+ for k in ctx.os.environ:
+ # DOCKER* environment variable are used by the docker client
+ # to know how to talk to the docker daemon.
+ if k.startswith("DOCKER"):
+ exports.append("export %s='%s'" % (k, ctx.os.environ[k]))
+ ctx.symlink(docker, "docker-bin")
+ ctx.file("docker.sh", "\n".join([
+ "#!/bin/bash",
+ "\n".join(exports),
+"""BIN="$0"
+while [ -L "${BIN}" ]; do
+ BIN="$(readlink "${BIN}")"
+done
+exec "${BIN%%.sh}-bin" "$@"
+"""]))
+ ctx.file("BUILD", "\n".join([
+ "sh_binary(",
+ " name = 'docker',",
+ " srcs = ['docker.sh'],",
+ " data = [':docker-bin'],",
+ " visibility = ['//visibility:public'],",
+ ")"]))
+
+docker_repository_ = repository_rule(_impl)
+
+def docker_repository():
+ """Declare a @docker repository that provide a docker binary."""
+ docker_repository_(name = "docker")
diff --git a/jenkins/test/test-support.sh b/jenkins/test/test-support.sh
new file mode 100644
index 0000000..6140be2
--- /dev/null
+++ b/jenkins/test/test-support.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Utilities to write integration test that run jenkins inside docker
+# and interrogate the jenkins master.
+
+DOCKER="${PYTHON_RUNFILES}/docker/docker"
+source "${PYTHON_RUNFILES}/io_bazel_ci/jenkins/test/common.sh"
+
+set -e
+
+# Machine on which the docker daemon is running, default is localhost.
+: ${DOCKER_SERVER:=127.0.0.1}
+
+setup() {
+ __test_support_containers="$(test_setup "${DOCKER_SERVER}" 0)"
+ __test_support_port=$(get_jenkins_port ${__test_support_containers})
+ __test_support_diagnostics=
+}
+
+teardown() {
+ local jenkins="$(f() { echo $1; }; f ${__test_support_containers})"
+ local logs="$(docker logs $jenkins)"
+ kill_containers ${__test_support_containers}
+ if [ -n "${__test_support_diagnostics}" ]; then
+ >&2 echo '
+*** LOG FROM THE JENKINS MASTER ***
+'"${logs}"'
+
+*** FAILURES SUMMARY ***
+'"${__test_support_diagnostics}
+"
+ exit 1
+ else
+ >&2 echo '
+
+*** ALL TESTS PASSED ***
+'
+ fi
+}
+
+# Utilities
+mcurl() {
+ local url="$1"
+ shift 1
+ # Since we are connecting to localhost and we tested the server is up and running,
+ # we limit the request to 1 seconds for connection and 10 seconds total
+ curl --connect-timeout 1 -m 10 -s "http://${DOCKER_SERVER}:${__test_support_port}${url}" "$@"
+}
+
+get_status_code() {
+ mcurl "$1" -I -L | grep '^HTTP/' | tail -1 | cut -d " " -f 2
+}
+
+diagnostics=
+report() {
+ echo "FAILED: $*" >&2
+ __test_support_diagnostics="${__test_support_diagnostics}
+$*"
+}
+
+test_status_code() {
+ local code="$(get_status_code "$1")"
+ if [ "$code" != "$2" ]; then
+ report "Got status $code while expecting $2 from $1"
+ else
+ echo "OK $1 returned $2"
+ fi
+}
+
+test_ok_status() {
+ test_status_code "$1" 200
+}
diff --git a/jenkins/test/test.sh b/jenkins/test/test.sh
new file mode 100755
index 0000000..5dba310
--- /dev/null
+++ b/jenkins/test/test.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Copyright 2016 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runs the jenkins test images inside the docker container.
+# Just a wrapper script to use the test support in run mode
+
+# So that the docker incremental loader knows where the runfiles are.
+export PYTHON_RUNFILES="${PYTHON_RUNFILES:-${BASH_SOURCE[0]}.runfiles}"
+cd "${PYTHON_RUNFILES}"
+PYTHON_RUNFILES="${PWD}"
+
+source "${PYTHON_RUNFILES}/io_bazel_ci/jenkins/test/common.sh"
+
+set -e
+
+# Port to serve Jenkins on, default is 8080.
+PORT=8080
+
+# Machine on which the docker daemon is running, default is localhost.
+: ${DOCKER_SERVER:=127.0.0.1}
+
+VOLUMES=()
+while getopts ":p:h:s:" opt; do
+ case "${opt}" in
+ p)
+ PORT="${OPTARG}"
+ ;;
+ h)
+ VOLUMES=(-v "${OPTARG}:/volumes/jenkins_home")
+ ;;
+ s)
+ VOLUMES=(-v "${OPTARG}:/volumes/secrets")
+ ;;
+ *)
+ echo "Usage: $0 [-p <port>] [-s </volumes/secrets>] [-h </volumes/jenkins_home>]" >&2
+ exit 1
+ ;;
+ esac
+done
+
+containers="$(test_setup "${DOCKER_SERVER}" "${PORT}" "${VOLUMES[@]}")"
+attach ${containers}
+kill_containers ${containers}
diff --git a/jenkins/test/test_url_answer.sh b/jenkins/test/test_url_answer.sh
new file mode 100755
index 0000000..2825f59
--- /dev/null
+++ b/jenkins/test/test_url_answer.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Simple test that run jenkins and verify that some content is there.
+
+# So that the docker incremental loader knows where the runfiles are.
+export PYTHON_RUNFILES="${PYTHON_RUNFILES:-${BASH_SOURCE[0]}.runfiles}"
+cd "${PYTHON_RUNFILES}"
+PYTHON_RUNFILES="${PWD}"
+
+source "${PYTHON_RUNFILES}/io_bazel_ci/jenkins/test/test-support.sh"
+setup
+
+test_ok_status "/job/Global/job/pipeline/"
+test_ok_status "/job/Github-Trigger/"
+
+test_ok_status "/job/rules_closure/"
+test_ok_status "/job/PR/job/rules_closure/"
+test_ok_status "/job/Global/job/rules_closure/"
+
+test_ok_status "/job/bazel-tests/"
+test_ok_status "/job/PR/job/bazel-tests/"
+test_ok_status "/job/CR/job/bazel-tests/"
+test_ok_status "/job/Global/job/bazel-tests/"
+
+teardown
