Moved jenkins.bzl to a subdirectory and split it
Change-Id: I2cfa48cbf6824b3dbd4f7c538fe4b523f5f73a33
diff --git a/jenkins/BUILD b/jenkins/BUILD
index 4124a2e..1fd6b5e 100644
--- a/jenkins/BUILD
+++ b/jenkins/BUILD
@@ -14,12 +14,9 @@
# Configuration for our Jenkins instance
load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build")
-load(":jenkins.bzl", "jenkins_node", "jenkins_build", "jenkins_job")
-load(":nodes.bzl", "jenkins_nodes", "jenkins_node_names")
+load("//jenkins/build_defs:jenkins.bzl", "jenkins_node", "jenkins_docker_build", "jenkins_job", "jenkins_nodes", "jenkins_node_names")
load("//jenkins/jobs:jobs.bzl", "JOBS_SUBSTITUTIONS", "STAGING_JOBS_SUBSTITUTIONS")
-exports_files(glob(["github-jobs*.tpl"]))
-
#
# Nodes
#
@@ -279,7 +276,7 @@
#
# Finally the Jenkins image
#
-jenkins_build(
+jenkins_docker_build(
name = "jenkins",
configs = [
":deploy",
@@ -305,7 +302,7 @@
visibility = ["//visibility:public"],
)
-jenkins_build(
+jenkins_docker_build(
name = "jenkins-staging",
configs = [
":darwin-x86_64-staging",
@@ -332,7 +329,7 @@
#
# A jenkins image for testing purpose
-jenkins_build(
+jenkins_docker_build(
name = "jenkins-test",
configs = [
":deploy",
diff --git a/jenkins/build_defs/BUILD b/jenkins/build_defs/BUILD
new file mode 100644
index 0000000..9d03353
--- /dev/null
+++ b/jenkins/build_defs/BUILD
@@ -0,0 +1 @@
+exports_files(glob(["github-jobs*.tpl"]))
diff --git a/jenkins/github-jobs-Gerrit.xml.tpl b/jenkins/build_defs/github-jobs-Gerrit.xml.tpl
similarity index 100%
rename from jenkins/github-jobs-Gerrit.xml.tpl
rename to jenkins/build_defs/github-jobs-Gerrit.xml.tpl
diff --git a/jenkins/github-jobs-PR.xml.tpl b/jenkins/build_defs/github-jobs-PR.xml.tpl
similarity index 100%
rename from jenkins/github-jobs-PR.xml.tpl
rename to jenkins/build_defs/github-jobs-PR.xml.tpl
diff --git a/jenkins/github-jobs.bat.tpl b/jenkins/build_defs/github-jobs.bat.tpl
similarity index 100%
rename from jenkins/github-jobs.bat.tpl
rename to jenkins/build_defs/github-jobs.bat.tpl
diff --git a/jenkins/github-jobs.sh.tpl b/jenkins/build_defs/github-jobs.sh.tpl
similarity index 100%
rename from jenkins/github-jobs.sh.tpl
rename to jenkins/build_defs/github-jobs.sh.tpl
diff --git a/jenkins/github-jobs.test-logs.bat.tpl b/jenkins/build_defs/github-jobs.test-logs.bat.tpl
similarity index 100%
rename from jenkins/github-jobs.test-logs.bat.tpl
rename to jenkins/build_defs/github-jobs.test-logs.bat.tpl
diff --git a/jenkins/github-jobs.test-logs.sh.tpl b/jenkins/build_defs/github-jobs.test-logs.sh.tpl
similarity index 100%
rename from jenkins/github-jobs.test-logs.sh.tpl
rename to jenkins/build_defs/github-jobs.test-logs.sh.tpl
diff --git a/jenkins/github-jobs.xml.tpl b/jenkins/build_defs/github-jobs.xml.tpl
similarity index 100%
rename from jenkins/github-jobs.xml.tpl
rename to jenkins/build_defs/github-jobs.xml.tpl
diff --git a/jenkins/build_defs/jenkins.bzl b/jenkins/build_defs/jenkins.bzl
new file mode 100644
index 0000000..53d45f7
--- /dev/null
+++ b/jenkins/build_defs/jenkins.bzl
@@ -0,0 +1,20 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Some definition to setup jenkins and build the corresponding docker images
+load(":jenkins_docker_build.bzl", "jenkins_docker_build")
+load(":jenkins_node.bzl", "jenkins_node")
+load(":jenkins_nodes.bzl", "jenkins_nodes", "jenkins_node_names")
+load(":jenkins_job.bzl", "jenkins_job", "bazel_git_job", "bazel_github_job")
+
diff --git a/jenkins/build_defs/jenkins_docker_build.bzl b/jenkins/build_defs/jenkins_docker_build.bzl
new file mode 100644
index 0000000..788b0db
--- /dev/null
+++ b/jenkins/build_defs/jenkins_docker_build.bzl
@@ -0,0 +1,65 @@
+# Copyright 2015 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Creation of the docker container for the jenkins master.
+
+load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build")
+load(":templates.bzl", "merge_files")
+load(":vars.bzl", "MAIL_SUBSTITUTIONS")
+
+def jenkins_docker_build(name, plugins = None, base = "//jenkins/base", configs = [],
+ jobs = [], substitutions = {}, visibility = None, tars = []):
+ """Build the docker image for the Jenkins instance."""
+ substitutions = substitutions + MAIL_SUBSTITUTIONS
+ # Expands config files in a tar ball
+ merge_files(
+ name = "%s-configs" % name,
+ srcs = configs,
+ directory = "/usr/share/jenkins/ref",
+ strip_prefixes = [
+ "jenkins/config",
+ "jenkins",
+ ],
+ substitutions = substitutions)
+
+ # Create the structures for jobs
+ merge_files(
+ name = "%s-jobs" % name,
+ srcs = jobs,
+ path_format = "jobs/{basename}/config.xml",
+ directory = "/usr/share/jenkins/ref",
+ )
+
+ ### FINAL IMAGE ###
+ docker_build(
+ name = name,
+ tars = [
+ ":%s-jobs" % name,
+ ":%s-configs" % name,
+ ] + tars,
+ # Workaround no way to specify owner in pkg_tar
+ # TODO(dmarting): use https://cr.bazel.build/10255 when it hits a release.
+ user = "root",
+ entrypoint = [
+ "/bin/tini",
+ "--",
+ "/bin/bash",
+ "-c",
+ "[ -d /opt/lib ] && chown -R jenkins /opt/lib; su jenkins -c /usr/local/bin/jenkins.sh",
+ ],
+ # End of workaround
+ base = base,
+ directory = "/",
+ visibility = visibility,
+ )
diff --git a/jenkins/build_defs/jenkins_job.bzl b/jenkins/build_defs/jenkins_job.bzl
new file mode 100644
index 0000000..4af6990
--- /dev/null
+++ b/jenkins/build_defs/jenkins_job.bzl
@@ -0,0 +1,182 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Jenkins job creation
+
+load(":templates.bzl", "expand_template")
+load(":vars.bzl", "MAIL_SUBSTITUTIONS")
+
+def jenkins_job(name, config, substitutions = {}, deps = [],
+ project='bazel', org='bazelbuild', git_url=None, project_url=None,
+ folder=None, platforms=[], test_platforms=["linux-x86_64"]):
+ """Create a job configuration on Jenkins.
+
+ Args:
+ name: the name of the job to create
+ config: the configuration file for the job
+ substitutions: additional substitutions to pass to the template generation
+ deps: list of dependencies (templates included by the config file)
+ project: the project name on github
+ org: the project organization on github, default 'bazelbuild'
+ git_url: the URL to the git project, defaulted to the Github URL
+ project_url: the project url, defaulted to the Git URL
+ platforms: platforms on which to run that job, default None,
+ test_platforms: platforms on which to run that job when inside of a
+ dockerized test, by default only 'linux-x86_64'
+ """
+ github_project = "%s/%s" % (org, project.lower())
+ github_url = "https://github.com/" + github_project
+ if not git_url:
+ git_url = github_url
+ if not project_url:
+ project_url = git_url
+ substitutions = substitutions + {
+ "GITHUB_URL": github_url,
+ "GIT_URL": git_url,
+ "GITHUB_PROJECT": github_project,
+ "PROJECT_URL": project_url,
+ "PLATFORMS": "\n".join(platforms),
+ } + MAIL_SUBSTITUTIONS
+ substitutions["SEND_EMAIL"] = "1"
+ expand_template(
+ name = name,
+ template = config,
+ out = "%s.xml" % name,
+ deps = deps,
+ substitutions = substitutions,
+ )
+ substitutions["SEND_EMAIL"] = "0"
+ expand_template(
+ name = name + "-staging",
+ template = config,
+ out = "%s-staging.xml" % name,
+ deps = deps,
+ substitutions = substitutions,
+ )
+
+ if test_platforms:
+ substitutions["PLATFORMS"] = "\n".join(test_platforms)
+ expand_template(
+ name = name + "-test",
+ template = config,
+ out = "%s-test.xml" % name,
+ deps = deps,
+ substitutions = substitutions,
+ )
+
+def bazel_git_job(**kwargs):
+ """Override bazel_github_job to test a project that is not on GitHub."""
+ kwargs["github_enabled"] = False
+ if not "git_url" in kwargs:
+ if not "project_url" in kwargs:
+ fail("Neither project_url nor git_url was specified")
+ kwargs["git_url"] = kwargs
+ bazel_github_job(**kwargs)
+
+def bazel_github_job(name, platforms=[], branch="master", project=None, org="google",
+ project_url=None, workspace=".", configure=[], git_url=None,
+ bazel_versions=["HEAD", "latest"],
+ tests=["//..."], targets=["//..."], substitutions={},
+ windows_configure=[],
+ windows_tests=["//..."], windows_targets=["//..."],
+ windows_tests_msys=["//..."], windows_targets_msys=["//..."],
+ test_opts=["--test_output=errors", "--build_tests_only"],
+ test_tag_filters=["-noci", "-manual"],
+ build_opts=["--verbose_failures"],
+ test_platforms=["linux-x86_64"],
+ enable_trigger=True,
+ gerrit_project=None,
+ enabled=True,
+ pr_enabled=True,
+ github_enabled=True,
+ run_sequential=False,
+ sauce_enabled=False):
+ """Create a generic github job configuration to build against Bazel head."""
+ if not project:
+ project = name
+ substitutions = substitutions + {
+ "WORKSPACE": workspace,
+ "PROJECT_NAME": project,
+ "BRANCH": branch,
+ "CONFIGURE": "\n".join(configure),
+ "WINDOWS_CONFIGURE": "\n".join(windows_configure),
+ "TEST_OPTS": " ".join(test_opts),
+ "TEST_TAG_FILTERS": ",".join(test_tag_filters),
+ "BUILD_OPTS": " ".join(build_opts),
+ "TESTS": " + ".join(tests),
+ "WINDOWS_TESTS": " ".join(windows_tests),
+ # TODO(pcloudy): remove *_MSYS attributes when we don't need MSYS anymore
+ "WINDOWS_TESTS_MSYS": " ".join(windows_tests_msys),
+ "BUILDS": " ".join(targets),
+ "WINDOWS_BUILDS": " ".join(windows_targets),
+ "WINDOWS_BUILDS_MSYS": " ".join(windows_targets_msys),
+ "BAZEL_VERSIONS": "\n".join(bazel_versions),
+ "disabled": str(not enabled).lower(),
+ "enable_trigger": str(enable_trigger and github_enabled).lower(),
+ "github": str(github_enabled),
+ "GERRIT_PROJECT": str(gerrit_project),
+ "RUN_SEQUENTIAL": str(run_sequential).lower(),
+ "SAUCE_ENABLED": str(sauce_enabled).lower(),
+ }
+
+ jenkins_job(
+ name = name,
+ config = "//jenkins/build_defs:github-jobs.xml.tpl",
+ deps = [
+ "//jenkins/build_defs:github-jobs.sh.tpl",
+ "//jenkins/build_defs:github-jobs.bat.tpl",
+ "//jenkins/build_defs:github-jobs.test-logs.sh.tpl",
+ "//jenkins/build_defs:github-jobs.test-logs.bat.tpl",
+ ],
+ substitutions=substitutions,
+ git_url=git_url,
+ project=project,
+ org=org,
+ project_url=project_url,
+ platforms=platforms,
+ test_platforms=test_platforms)
+ substitutions["BAZEL_VERSIONS"] = "\n".join([
+ v for v in bazel_versions if not v.startswith("HEAD")])
+ if pr_enabled:
+ jenkins_job(
+ name = "PR-" + name,
+ config = "//jenkins/build_defs:github-jobs-PR.xml.tpl",
+ deps = [
+ "//jenkins/build_defs:github-jobs.sh.tpl",
+ "//jenkins/build_defs:github-jobs.bat.tpl",
+ "//jenkins/build_defs:github-jobs.test-logs.sh.tpl",
+ "//jenkins/build_defs:github-jobs.test-logs.bat.tpl",
+ ],
+ substitutions=substitutions,
+ project=project,
+ org=org,
+ project_url=project_url,
+ platforms=platforms,
+ test_platforms=test_platforms)
+ if gerrit_project != None:
+ jenkins_job(
+ name = "Gerrit-" + name,
+ config = "//jenkins/build_defs:github-jobs-Gerrit.xml.tpl",
+ deps = [
+ "//jenkins/build_defs:github-jobs.sh.tpl",
+ "//jenkins/build_defs:github-jobs.bat.tpl",
+ "//jenkins/build_defs:github-jobs.test-logs.sh.tpl",
+ "//jenkins/build_defs:github-jobs.test-logs.bat.tpl",
+ ],
+ substitutions=substitutions,
+ project=project,
+ org=org,
+ project_url=project_url,
+ platforms=platforms,
+ test_platforms=test_platforms)
diff --git a/jenkins/build_defs/jenkins_node.bzl b/jenkins/build_defs/jenkins_node.bzl
new file mode 100644
index 0000000..9c4c8fa
--- /dev/null
+++ b/jenkins/build_defs/jenkins_node.bzl
@@ -0,0 +1,90 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Setup jenkins and build the corresponding docker images
+
+load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build")
+load(":templates.bzl", "expand_template")
+
+JENKINS_SERVER = "http://jenkins:80"
+
+def jenkins_node(name, remote_fs = "/home/ci", num_executors = 1, mode = "NORMAL",
+ labels = [], docker_base = None, preference = 1,
+ visibility = None):
+ """Create a node configuration on Jenkins, with possible docker image.
+
+ Args:
+ name: Name of the node on Jenkins.
+ remote_fs: path to the home of the Jenkins user.
+ num_executors: number of executors (i.e. concurrent build) this machine can have.
+ mode: NORMAL for "Utilize this node as much as possible"
+ EXCLUSIVE for "Only build jobs with label restrictions matching this node"
+ labels: list of Jenkins labels for this node (the node name is always added).
+ docker_base: base for the corresponding docker image to create if we should create one
+ (if docker_base is not specified, then a corresponding machine should be configured
+ to connect to the Jenkins master).
+ preference: A preference factor, if a node as a factor of 1 and another a factor of
+ 4, then the second one will be scheduled 4 time more jobs than the first one.
+ visibility: rule visibility.
+ """
+ native.genrule(
+ name = name,
+ cmd = """cat >$@ <<'EOF'
+<?xml version='1.0' encoding='UTF-8'?>
+<slave>
+ <name>%s</name>
+ <description></description>
+ <remoteFS>%s</remoteFS>
+ <numExecutors>%s</numExecutors>
+ <mode>%s</mode>
+ <retentionStrategy class="hudson.slaves.RetentionStrategy$$Always"/>
+ <launcher class="hudson.slaves.JNLPLauncher"/>
+ <label>%s</label>
+ <nodeProperties>
+ <jp.ikedam.jenkins.plugins.scoringloadbalancer.preferences.BuildPreferenceNodeProperty plugin="scoring-load-balancer@1.0.1">
+ <preference>%s</preference>
+ </jp.ikedam.jenkins.plugins.scoringloadbalancer.preferences.BuildPreferenceNodeProperty>
+ </nodeProperties>
+</slave>
+EOF
+""" % (name, remote_fs, num_executors, mode, " ".join([name] + labels), preference),
+ outs = ["nodes/%s/config.xml" % name],
+ visibility = visibility,
+ )
+ if docker_base:
+ # Generate docker image startup script
+ expand_template(
+ name = name + ".docker-launcher",
+ out = name + ".docker-launcher.sh",
+ template = "slave_setup.sh",
+ substitutions = {
+ "NODE_NAME": name,
+ "HOME_FS": remote_fs,
+ "JENKINS_SERVER": JENKINS_SERVER,
+ },
+ executable = True,
+ )
+ # Generate docker image
+ docker_build(
+ name = name + ".docker",
+ base = docker_base,
+ volumes = [remote_fs],
+ files = [":%s.docker-launcher.sh" % name],
+ data_path = ".",
+ entrypoint = [
+ "/bin/bash",
+ "/%s.docker-launcher.sh" % name,
+ ],
+ visibility = visibility,
+ )
diff --git a/jenkins/nodes.bzl b/jenkins/build_defs/jenkins_nodes.bzl
similarity index 97%
rename from jenkins/nodes.bzl
rename to jenkins/build_defs/jenkins_nodes.bzl
index 7704d87..80a805f 100644
--- a/jenkins/nodes.bzl
+++ b/jenkins/build_defs/jenkins_nodes.bzl
@@ -13,7 +13,7 @@
# limitations under the License.
# Macros to ease the creation of machines
-load(":jenkins.bzl", "jenkins_node")
+load(":jenkins_node.bzl", "jenkins_node")
def jenkins_node_names(name, count):
"""Returns the names for `count` production jenkins node prefixed by `name`."""
diff --git a/jenkins/build_defs/templates.bzl b/jenkins/build_defs/templates.bzl
new file mode 100644
index 0000000..a25d84b
--- /dev/null
+++ b/jenkins/build_defs/templates.bzl
@@ -0,0 +1,202 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Rules for templating / files layout
+
+def _expand_template_impl(ctx):
+ """Simply spawn the template-engine in a rule."""
+ variables = [
+ "--variable=%s=%s" % (k, ctx.attr.substitutions[k])
+ for k in ctx.attr.substitutions
+ ]
+ imports = [
+ "--imports=%s=%s" % (ctx.attr.deps[i].label, ctx.files.deps[i].path)
+ for i in range(0, len(ctx.attr.deps))
+ ]
+ ctx.action(
+ executable = ctx.executable._engine,
+ arguments = [
+ "--executable" if ctx.attr.executable else "--noexecutable",
+ "--template=%s" % ctx.file.template.path,
+ "--output=%s" % ctx.outputs.out.path,
+ ] + variables + imports,
+ inputs = [ctx.file.template] + ctx.files.deps,
+ outputs = [ctx.outputs.out],
+ )
+
+expand_template = rule(
+ attrs = {
+ "template": attr.label(
+ mandatory = True,
+ allow_files = True,
+ single_file = True,
+ ),
+ "deps": attr.label_list(default = [], allow_files = True),
+ "substitutions": attr.string_dict(mandatory = True),
+ "out": attr.output(mandatory = True),
+ "executable": attr.bool(default = True),
+ "_engine": attr.label(
+ default = Label("//templating:template_engine"),
+ executable = True,
+ cfg="host"),
+ },
+ implementation = _expand_template_impl,
+)
+"""Expand a jinja2 template file.
+
+This rules expands the file given in template, into the file given by out.
+
+Args:
+ template: The template file to expand.
+ deps: additional files to expand, they will be accessible as imports[label]
+ in the template environment. If a file ends with .tpl, it is considered
+ a template itself and will be expanded.
+ deps_aliases: a dictionary of name to label. Each label in that dictionary
+ should be present in the deps attribute, and will be make accessible as
+ imports[name] in the template environment.
+ substitutions: a dictionary of key => values that will appear as variables.key
+ in the template environment.
+ out: the name of the output file to generate.
+ executable: mark the result as excutable if set to True.
+"""
+
+def _strip_prefix(path, prefixes):
+ for prefix in prefixes:
+ if path.startswith(prefix):
+ return path[len(prefix):]
+ return path
+
+def _strip_suffix(path, suffixes):
+ for suffix in suffixes:
+ if path.endswith(suffix):
+ return path[:-len(suffix)]
+ return path
+
+def _dest_path(f, strip_prefixes, strip_suffixes):
+ """Returns the short path of f, stripped of strip_prefixes and strip_suffixes."""
+ return _strip_suffix(_strip_prefix(f.short_path, strip_prefixes), strip_suffixes)
+
+def _format_path(path_format, path):
+ dirsep = path.rfind("/")
+ dirname = path[:dirsep] if dirsep > 0 else ""
+ basename = path[dirsep+1:] if dirsep > 0 else path
+ extsep = basename.rfind(".")
+ extension = basename[extsep+1:] if extsep > 0 else ""
+ basename = basename[:extsep] if extsep > 0 else basename
+ return path_format.format(
+ path=path,
+ dirname=dirname,
+ basename=basename,
+ extension=extension
+ )
+
+def _append_inputs(args, inputs, f, path, path_format):
+ args.append("--file=%s=%s" % (
+ f.path,
+ _format_path(path_format, path)
+ ))
+ inputs.append(f)
+
+def _merge_files_impl(ctx):
+ """Merge a list of config files in a tar ball with the correct layout."""
+ output = ctx.outputs.out
+ build_tar = ctx.executable._build_tar
+ inputs = []
+ args = [
+ "--output=" + output.path,
+ "--directory=" + ctx.attr.directory,
+ "--mode=0644",
+ ]
+ variables = [
+ "--variable=%s=%s" % (k, ctx.attr.substitutions[k])
+ for k in ctx.attr.substitutions
+ ]
+ for f in ctx.files.srcs:
+ path = _dest_path(f, ctx.attr.strip_prefixes, ctx.attr.strip_suffixes)
+ if path.endswith(ctx.attr.template_extension):
+ path = path[:-4]
+ f2 = ctx.new_file(ctx.label.name + "/" + path)
+ ctx.action(
+ executable = ctx.executable._engine,
+ arguments = [
+ "--template=%s" % f.path,
+ "--output=%s" % f2.path,
+ "--noescape_xml",
+ ] + variables,
+ inputs = [f],
+ outputs = [f2],
+ )
+ _append_inputs(args, inputs, f2, path, ctx.attr.path_format)
+ else:
+ _append_inputs(args, inputs, f, path, ctx.attr.path_format)
+ ctx.action(
+ executable = build_tar,
+ arguments = args,
+ inputs = inputs,
+ outputs = [output],
+ mnemonic="MergeFiles"
+ )
+
+merge_files = rule(
+ attrs = {
+ "srcs": attr.label_list(allow_files=True),
+ "template_extension": attr.string(default=".tpl"),
+ "directory": attr.string(default="/"),
+ "strip_prefixes": attr.string_list(default=[]),
+ "strip_suffixes": attr.string_list(default=["-staging", "-test"]),
+ "substitutions": attr.string_dict(default={}),
+ "path_format": attr.string(default="{path}"),
+ "_build_tar": attr.label(
+ default=Label("@bazel_tools//tools/build_defs/pkg:build_tar"),
+ cfg="host",
+ executable=True,
+ allow_files=True),
+ "_engine": attr.label(
+ cfg="host",
+ default = Label("//templating:template_engine"),
+ executable = True),
+ },
+ outputs = {"out": "%{name}.tar"},
+ implementation = _merge_files_impl,
+)
+"""Merge a set of files in a single tarball.
+
+This rule merge a set of files into one tarball, each file will appear in the
+tarball as a file determined by path_format, strip_prefixes and directory.
+
+Outputs:
+ <name>.tar: the tarball containing all the files in srcs.
+
+Args:
+ srcs: The list of files to merge. If a file is ending with ".tpl" (see
+ template_extension), it will get expanded like a template passed to
+ expand_template.
+ template_extension: extension of files to be considered as template, ".tpl"
+ by default.
+ directory: base directory for all the files in the resulting tarball.
+ strip_prefixes: list of prefixes to strip from the path of the srcs to obtain
+ the final path (see path_format).
+ strip_suffixes: list of suffixes to strip from the path of the srcs to obtain
+ the final path (see path_format).
+ substitutions: map of substitutions to make available during template
+ expansion. Values of that map will be available as "variables.name" in
+ the template environment.
+ path_format: format of the final files. Each file will appear in the final
+ tarball under "{directory}/{path_format}" where the following string of
+ path_format are replaced:
+ {path}: path of the input file, removed from prefixes and suffixes,
+ {dirname}: directory name of path,
+ {basename}: base filename of path,
+ {extension}: extension of path
+"""
diff --git a/jenkins/build_defs/vars.bzl b/jenkins/build_defs/vars.bzl
new file mode 100644
index 0000000..3892447
--- /dev/null
+++ b/jenkins/build_defs/vars.bzl
@@ -0,0 +1,21 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Global constants for jenkins jobs substitutions
+
+MAIL_SUBSTITUTIONS = {
+ "BAZEL_BUILD_RECIPIENT": "bazel-ci@googlegroups.com",
+ "BAZEL_RELEASE_RECIPIENT": "bazel-discuss+release@googlegroups.com",
+ "SENDER_EMAIL": "noreply@bazel.io",
+}
diff --git a/jenkins/jenkins.bzl b/jenkins/jenkins.bzl
deleted file mode 100644
index 6410d31..0000000
--- a/jenkins/jenkins.bzl
+++ /dev/null
@@ -1,446 +0,0 @@
-# Copyright 2015 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Some definition to setup jenkins and build the corresponding docker images
-
-load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build")
-load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar")
-
-JENKINS_PORT = 80
-JENKINS_HOST = "jenkins"
-
-MAILS_SUBSTITUTIONS = {
- "BAZEL_BUILD_RECIPIENT": "bazel-ci@googlegroups.com",
- "BAZEL_RELEASE_RECIPIENT": "bazel-discuss+release@googlegroups.com",
- "SENDER_EMAIL": "noreply@bazel.io",
-}
-
-def expand_template_impl(ctx):
- """Simply spawn the template-engine in a rule."""
- variables = [
- "--variable=%s=%s" % (k, ctx.attr.substitutions[k])
- for k in ctx.attr.substitutions
- ]
- imports = [
- "--imports=%s=%s" % (ctx.attr.deps[i].label, ctx.files.deps[i].path)
- for i in range(0, len(ctx.attr.deps))
- ]
- ctx.action(
- executable = ctx.executable._engine,
- arguments = [
- "--executable" if ctx.attr.executable else "--noexecutable",
- "--template=%s" % ctx.file.template.path,
- "--output=%s" % ctx.outputs.out.path,
- ] + variables + imports,
- inputs = [ctx.file.template] + ctx.files.deps,
- outputs = [ctx.outputs.out],
- )
-
-expand_template = rule(
- attrs = {
- "template": attr.label(
- mandatory = True,
- allow_files = True,
- single_file = True,
- ),
- "deps": attr.label_list(default = [], allow_files = True),
- "substitutions": attr.string_dict(mandatory = True),
- "out": attr.output(mandatory = True),
- "executable": attr.bool(default = True),
- "_engine": attr.label(
- default = Label("//templating:template_engine"),
- executable = True,
- cfg="host"),
- },
- implementation = expand_template_impl,
-)
-
-def _dest_path(f, strip_prefixes):
- """Returns the short path of f, stripped of strip_prefix."""
- for strip_prefix in strip_prefixes:
- if f.short_path.startswith(strip_prefix):
- return f.short_path[len(strip_prefix):]
- return f.short_path
-
-def _format_path(path_format, path):
- dirsep = path.rfind("/")
- dirname = path[:dirsep] if dirsep > 0 else ""
- basename = path[dirsep+1:] if dirsep > 0 else path
- extsep = basename.rfind(".")
- extension = basename[extsep+1:] if extsep > 0 else ""
- basename = basename[:extsep] if extsep > 0 else basename
- flavor = ""
- if basename.endswith("-staging"):
- basename = basename[:-8]
- flavor = "staging"
- elif basename.endswith("-test"):
- basename = basename[:-5]
- flavor = "test"
- return path_format.format(
- path=path,
- dirname=dirname,
- basename=basename,
- flavor=flavor,
- extension=extension
- )
-
-def _append_inputs(args, inputs, f, path, path_format):
- args.append("--file=%s=%s" % (
- f.path,
- _format_path(path_format, path)
- ))
- inputs.append(f)
-
-def _merge_files_impl(ctx):
- """Merge a list of config files in a tar ball with the correct layout."""
- output = ctx.outputs.out
- build_tar = ctx.executable._build_tar
- inputs = []
- args = [
- "--output=" + output.path,
- "--directory=" + ctx.attr.directory,
- "--mode=0644",
- ]
- variables = [
- "--variable=%s=%s" % (k, ctx.attr.substitutions[k])
- for k in ctx.attr.substitutions
- ]
- for f in ctx.files.srcs:
- path = _dest_path(f, ctx.attr.strip_prefixes)
- if path.endswith(".tpl"):
- path = path[:-4]
- f2 = ctx.new_file(ctx.label.name + "/" + path)
- ctx.action(
- executable = ctx.executable._engine,
- arguments = [
- "--template=%s" % f.path,
- "--output=%s" % f2.path,
- "--noescape_xml",
- ] + variables,
- inputs = [f],
- outputs = [f2],
- )
- _append_inputs(args, inputs, f2, path, ctx.attr.path_format)
- else:
- _append_inputs(args, inputs, f, path, ctx.attr.path_format)
- ctx.action(
- executable = build_tar,
- arguments = args,
- inputs = inputs,
- outputs = [output],
- mnemonic="MergeFiles"
- )
-
-_merge_files = rule(
- attrs = {
- "srcs": attr.label_list(allow_files=True),
- "template_extension": attr.string(default=".tpl"),
- "directory": attr.string(default="/"),
- "strip_prefixes": attr.string_list(default=[]),
- "substitutions": attr.string_dict(default={}),
- "path_format": attr.string(default="{path}"),
- "_build_tar": attr.label(
- default=Label("@bazel_tools//tools/build_defs/pkg:build_tar"),
- cfg="host",
- executable=True,
- allow_files=True),
- "_engine": attr.label(
- cfg="host",
- default = Label("//templating:template_engine"),
- executable = True),
- },
- outputs = {"out": "%{name}.tar"},
- implementation = _merge_files_impl,
-)
-
-def jenkins_job(name, config, substitutions = {}, deps = [],
- project='bazel', org='bazelbuild', git_url=None, project_url=None,
- platforms=[], test_platforms=["linux-x86_64"]):
- """Create a job configuration on Jenkins.
-
- Args:
- name: the name of the job to create
- config: the configuration file for the job
- substitutions: additional substitutions to pass to the template generation
- deps: list of dependencies (templates included by the config file)
- project: the project name on github
- org: the project organization on github, default 'bazelbuild'
- git_url: the URL to the git project, defaulted to the Github URL
- project_url: the project url, defaulted to the Git URL
- platforms: platforms on which to run that job, default None,
- test_platforms: platforms on which to run that job when inside of a
- dockerized test, by default only 'linux-x86_64'
- """
- github_project = "%s/%s" % (org, project.lower())
- github_url = "https://github.com/" + github_project
- if not git_url:
- git_url = github_url
- if not project_url:
- project_url = git_url
- substitutions = substitutions + {
- "GITHUB_URL": github_url,
- "GIT_URL": git_url,
- "GITHUB_PROJECT": github_project,
- "PROJECT_URL": project_url,
- "PLATFORMS": "\n".join(platforms),
- } + MAILS_SUBSTITUTIONS
- substitutions["SEND_EMAIL"] = "1"
- expand_template(
- name = name,
- template = config,
- out = "%s.xml" % name,
- deps = deps,
- substitutions = substitutions,
- )
- substitutions["SEND_EMAIL"] = "0"
- expand_template(
- name = name + "-staging",
- template = config,
- out = "%s-staging.xml" % name,
- deps = deps,
- substitutions = substitutions,
- )
-
- if test_platforms:
- substitutions["PLATFORMS"] = "\n".join(test_platforms)
- expand_template(
- name = name + "-test",
- template = config,
- out = "%s-test.xml" % name,
- deps = deps,
- substitutions = substitutions,
- )
-
-def bazel_git_job(**kwargs):
- """Override bazel_github_job to test a project that is not on GitHub."""
- kwargs["github_enabled"] = False
- if not "git_url" in kwargs:
- if not "project_url" in kwargs:
- fail("Neither project_url nor git_url was specified")
- kwargs["git_url"] = kwargs
- bazel_github_job(**kwargs)
-
-def bazel_github_job(name, platforms=[], branch="master", project=None, org="google",
- project_url=None, workspace=".", configure=[], git_url=None,
- bazel_versions=["HEAD", "latest"],
- tests=["//..."], targets=["//..."], substitutions={},
- windows_configure=[],
- windows_tests=["//..."], windows_targets=["//..."],
- windows_tests_msys=["//..."], windows_targets_msys=["//..."],
- test_opts=["--test_output=errors", "--build_tests_only"],
- test_tag_filters=["-noci", "-manual"],
- build_opts=["--verbose_failures"],
- test_platforms=["linux-x86_64"],
- enable_trigger=True,
- gerrit_project=None,
- enabled=True,
- pr_enabled=True,
- github_enabled=True,
- run_sequential=False,
- sauce_enabled=False):
- """Create a generic github job configuration to build against Bazel head."""
- if not project:
- project = name
- substitutions = substitutions + {
- "WORKSPACE": workspace,
- "PROJECT_NAME": project,
- "BRANCH": branch,
- "CONFIGURE": "\n".join(configure),
- "WINDOWS_CONFIGURE": "\n".join(windows_configure),
- "TEST_OPTS": " ".join(test_opts),
- "TEST_TAG_FILTERS": ",".join(test_tag_filters),
- "BUILD_OPTS": " ".join(build_opts),
- "TESTS": " + ".join(tests),
- "WINDOWS_TESTS": " ".join(windows_tests),
- # TODO(pcloudy): remove *_MSYS attributes when we don't need MSYS anymore
- "WINDOWS_TESTS_MSYS": " ".join(windows_tests_msys),
- "BUILDS": " ".join(targets),
- "WINDOWS_BUILDS": " ".join(windows_targets),
- "WINDOWS_BUILDS_MSYS": " ".join(windows_targets_msys),
- "BAZEL_VERSIONS": "\n".join(bazel_versions),
- "disabled": str(not enabled).lower(),
- "enable_trigger": str(enable_trigger and github_enabled).lower(),
- "github": str(github_enabled),
- "GERRIT_PROJECT": str(gerrit_project),
- "RUN_SEQUENTIAL": str(run_sequential).lower(),
- "SAUCE_ENABLED": str(sauce_enabled).lower(),
- }
-
- jenkins_job(
- name = name,
- config = "//jenkins:github-jobs.xml.tpl",
- deps = [
- "//jenkins:github-jobs.sh.tpl",
- "//jenkins:github-jobs.bat.tpl",
- "//jenkins:github-jobs.test-logs.sh.tpl",
- "//jenkins:github-jobs.test-logs.bat.tpl",
- ],
- substitutions=substitutions,
- git_url=git_url,
- project=project,
- org=org,
- project_url=project_url,
- platforms=platforms,
- test_platforms=test_platforms)
- substitutions["BAZEL_VERSIONS"] = "\n".join([
- v for v in bazel_versions if not v.startswith("HEAD")])
- if pr_enabled:
- jenkins_job(
- name = "PR-" + name,
- config = "//jenkins:github-jobs-PR.xml.tpl",
- deps = [
- "//jenkins:github-jobs.sh.tpl",
- "//jenkins:github-jobs.bat.tpl",
- "//jenkins:github-jobs.test-logs.sh.tpl",
- "//jenkins:github-jobs.test-logs.bat.tpl",
- ],
- substitutions=substitutions,
- project=project,
- org=org,
- project_url=project_url,
- platforms=platforms,
- test_platforms=test_platforms)
- if gerrit_project != None:
- jenkins_job(
- name = "Gerrit-" + name,
- config = "//jenkins:github-jobs-Gerrit.xml.tpl",
- deps = [
- "//jenkins:github-jobs.sh.tpl",
- "//jenkins:github-jobs.bat.tpl",
- "//jenkins:github-jobs.test-logs.sh.tpl",
- "//jenkins:github-jobs.test-logs.bat.tpl",
- ],
- substitutions=substitutions,
- project=project,
- org=org,
- project_url=project_url,
- platforms=platforms,
- test_platforms=test_platforms)
-
-
-def jenkins_node(name, remote_fs = "/home/ci", num_executors = 1, mode = "NORMAL",
- labels = [], docker_base = None, preference = 1,
- visibility = None):
- """Create a node configuration on Jenkins, with possible docker image.
-
- Args:
- name: Name of the node on Jenkins.
- remote_fs: path to the home of the Jenkins user.
- num_executors: number of executors (i.e. concurrent build) this machine can have.
- mode: NORMAL for "Utilize this node as much as possible"
- EXCLUSIVE for "Only build jobs with label restrictions matching this node"
- labels: list of Jenkins labels for this node (the node name is always added).
- docker_base: base for the corresponding docker image to create if we should create one
- (if docker_base is not specified, then a corresponding machine should be configured
- to connect to the Jenkins master).
- preference: A preference factor, if a node as a factor of 1 and another a factor of
- 4, then the second one will be scheduled 4 time more jobs than the first one.
- visibility: rule visibility.
- """
- native.genrule(
- name = name,
- cmd = """cat >$@ <<'EOF'
-<?xml version='1.0' encoding='UTF-8'?>
-<slave>
- <name>%s</name>
- <description></description>
- <remoteFS>%s</remoteFS>
- <numExecutors>%s</numExecutors>
- <mode>%s</mode>
- <retentionStrategy class="hudson.slaves.RetentionStrategy$$Always"/>
- <launcher class="hudson.slaves.JNLPLauncher"/>
- <label>%s</label>
- <nodeProperties>
- <jp.ikedam.jenkins.plugins.scoringloadbalancer.preferences.BuildPreferenceNodeProperty plugin="scoring-load-balancer@1.0.1">
- <preference>%s</preference>
- </jp.ikedam.jenkins.plugins.scoringloadbalancer.preferences.BuildPreferenceNodeProperty>
- </nodeProperties>
-</slave>
-EOF
-""" % (name, remote_fs, num_executors, mode, " ".join([name] + labels), preference),
- outs = ["nodes/%s/config.xml" % name],
- visibility = visibility,
- )
- if docker_base:
- # Generate docker image startup script
- expand_template(
- name = name + ".docker-launcher",
- out = name + ".docker-launcher.sh",
- template = "slave_setup.sh",
- substitutions = {
- "NODE_NAME": name,
- "HOME_FS": remote_fs,
- "JENKINS_SERVER": "http://%s:%s" % (JENKINS_HOST, JENKINS_PORT),
- },
- executable = True,
- )
- # Generate docker image
- docker_build(
- name = name + ".docker",
- base = docker_base,
- volumes = [remote_fs],
- files = [":%s.docker-launcher.sh" % name],
- data_path = ".",
- entrypoint = [
- "/bin/bash",
- "/%s.docker-launcher.sh" % name,
- ],
- visibility = visibility,
- )
-
-def jenkins_build(name, plugins = None, base = "//jenkins/base", configs = [],
- jobs = [], substitutions = {}, visibility = None, tars = []):
- """Build the docker image for the Jenkins instance."""
- substitutions = substitutions + MAILS_SUBSTITUTIONS
- # Expands config files in a tar ball
- _merge_files(
- name = "%s-configs" % name,
- srcs = configs,
- directory = "/usr/share/jenkins/ref",
- strip_prefixes = [
- "jenkins/config",
- "jenkins",
- ],
- substitutions = substitutions)
-
- # Create the structures for jobs
- _merge_files(
- name = "%s-jobs" % name,
- srcs = jobs,
- path_format = "jobs/{basename}/config.xml",
- directory = "/usr/share/jenkins/ref",
- )
-
- ### FINAL IMAGE ###
- docker_build(
- name = name,
- tars = [
- ":%s-jobs" % name,
- ":%s-configs" % name,
- ] + tars,
- # Workaround no way to specify owner in pkg_tar
- # TODO(dmarting): use https://cr.bazel.build/10255 when it hits a release.
- user = "root",
- entrypoint = [
- "/bin/tini",
- "--",
- "/bin/bash",
- "-c",
- "[ -d /opt/lib ] && chown -R jenkins /opt/lib; su jenkins -c /usr/local/bin/jenkins.sh",
- ],
- # End of workaround
- base = base,
- directory = "/",
- visibility = visibility,
- )
diff --git a/jenkins/jobs/BUILD b/jenkins/jobs/BUILD
index d53d133..ca5a5ef 100644
--- a/jenkins/jobs/BUILD
+++ b/jenkins/jobs/BUILD
@@ -1,4 +1,4 @@
-load("//jenkins:jenkins.bzl", "jenkins_job", "bazel_github_job", "bazel_git_job")
+load("//jenkins/build_defs:jenkins.bzl", "jenkins_job", "bazel_github_job", "bazel_git_job")
load(
":jobs.bzl",
"LINUX_PLATFORMS",
