Refactor build for JDK 7
Now the JDK 7 tuning happens all in Bazel, removing logic
from the CI script. It uses remote repositories to access
JDK 7 dependencies.
--
Change-Id: Iff590c6642ca5b2343aa15096f8fd837d1c80787
Reviewed-on: https://bazel-review.googlesource.com/#/c/3327
MOS_MIGRATED_REVID=119634530
diff --git a/WORKSPACE b/WORKSPACE
index 05c5117..ed50f77 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -23,6 +23,22 @@
actual = "//:dummy",
)
+# For building for JDK 7
+# This is a JDK 7 JavaBuilder from release 0.1.0.
+# Do not use the skylark bootstrapped version of JavaBuilder
+http_file(
+ name = "io_bazel_javabuilder_jdk7",
+ # This was built from Bazel 0.1.0 (after ./compile.sh) and JDK 7.
+ url = "https://storage.googleapis.com/bazel/0.1.0/JavaBuilder_deploy.jar",
+ sha256 = "374f07be2f73ec170ef6bdd8057530e6842cb0cbc7a240caccbd7b80be239366",
+)
+
+http_file(
+ name = "io_bazel_javac_jdk7",
+ url = "https://github.com/bazelbuild/bazel/blob/0.1.0/third_party/java/jdk/langtools/javac.jar?raw=true",
+ sha256 = "e063cf60cf9a1d24acd043f55dc605da559b913793c6a4e51c5ddbc4f5d494df",
+)
+
# For tools/cpp/test/...
load("//tools/cpp/test:docker_repository.bzl", "docker_repository")
docker_repository()
diff --git a/compile.sh b/compile.sh
index 56299b8..da75ec0 100755
--- a/compile.sh
+++ b/compile.sh
@@ -96,8 +96,8 @@
new_step 'Building Bazel with Bazel'
display "."
log "Building output/bazel"
- bazel_build "src:bazel${EXE_EXT}"
- cp -f "bazel-bin/src/bazel${EXE_EXT}" "output/bazel${EXE_EXT}"
+ bazel_build "src:bazel${BAZEL_FLAVOUR}${EXE_EXT}"
+ cp -f "bazel-bin/src/bazel${BAZEL_FLAVOUR}${EXE_EXT}" "output/bazel${EXE_EXT}"
chmod 0755 "output/bazel${EXE_EXT}"
BAZEL="$(pwd)/output/bazel${EXE_EXT}"
fi
@@ -155,7 +155,6 @@
$BAZEL --bazelrc=${BAZELRC} --nomaster_bazelrc test \
--test_tag_filters="${BAZEL_TEST_FILTERS-}" \
--build_tests_only \
- --nolegacy_bazel_java_test \
${EXTRA_BAZEL_ARGS} \
--javacopt="-source ${JAVA_VERSION} -target ${JAVA_VERSION}" \
-k --test_output=errors //src/... //third_party/ijar/... //scripts/... \
diff --git a/scripts/bootstrap/bootstrap.sh b/scripts/bootstrap/bootstrap.sh
index 2fea800..3fae302 100755
--- a/scripts/bootstrap/bootstrap.sh
+++ b/scripts/bootstrap/bootstrap.sh
@@ -28,21 +28,33 @@
EMBED_LABEL_ARG=(--stamp --embed_label "${EMBED_LABEL}")
fi
-: ${JAVA_VERSION:="1.8"}
-
-: ${BAZEL_ARGS:=--singlejar_top=//src/java_tools/singlejar:bootstrap_deploy.jar \
- --javabuilder_top=//src/java_tools/buildjar:bootstrap_deploy.jar \
- --genclass_top=//src/java_tools/buildjar:bootstrap_genclass_deploy.jar \
- --ijar_top=//third_party/ijar \
+COMMON_BAZEL_ARGS="--singlejar_top=//src/java_tools/singlejar:bootstrap_deploy.jar \
+ --genclass_top=//src/java_tools/buildjar:bootstrap_genclass_deploy.jar \
+ --ijar_top=//third_party/ijar"
+if [[ "${JAVA_VERSION-}" == 1.7 ]]; then
+ # Set the various arguments when JDK 7 is required (deprecated).
+ # This setting is here to continue to build binary release of Bazel
+ # for JDK 7. We will drop this method and JDK 7 support when our
+ # ci system turn red on this one.
+ : ${BAZEL_ARGS:=--java_toolchain=//tools/jdk:toolchain7 \
+ --javabuilder_top=@io_bazel_javabuilder_jdk7//file \
+ --java_langtools=@io_bazel_javac_jdk7//file \
+ ${COMMON_BAZEL_ARGS} \
+ "${EXTRA_BAZEL_ARGS:-}"}
+ : ${BAZEL_FLAVOUR:="-jdk7"}
+else
+ : ${BAZEL_ARGS:=--javabuilder_top=//src/java_tools/buildjar:bootstrap_deploy.jar \
--strategy=Javac=worker --worker_quit_after_build \
--genrule_strategy=standalone --spawn_strategy=standalone \
+ ${COMMON_BAZEL_ARGS} \
"${EXTRA_BAZEL_ARGS:-}"}
+ : ${BAZEL_FLAVOUR:=""}
+fi
if [ -z "${BAZEL-}" ]; then
function bazel_build() {
bootstrap_build ${BAZEL_ARGS-} \
--verbose_failures \
- --javacopt="-source ${JAVA_VERSION} -target ${JAVA_VERSION}" \
"${EMBED_LABEL_ARG[@]}" \
"${@}"
}
@@ -51,7 +63,6 @@
${BAZEL} --bazelrc=${BAZELRC} build \
${BAZEL_ARGS-} \
--verbose_failures \
- --javacopt="-source ${JAVA_VERSION} -target ${JAVA_VERSION}" \
"${EMBED_LABEL_ARG[@]}" \
"${@}"
}
@@ -77,7 +88,7 @@
function bootstrap_test() {
local BAZEL_BIN=$1
local BAZEL_SUM=$2
- local BAZEL_TARGET=${3:-src:bazel}
+ local BAZEL_TARGET=${3:-src:bazel${BAZEL_FLAVOUR}}
[ -x "${BAZEL_BIN}" ] || fail "syntax: bootstrap bazel-binary"
run ${BAZEL_BIN} --nomaster_bazelrc --bazelrc=${BAZELRC} clean \
--expunge || return $?
@@ -85,7 +96,6 @@
${EXTRA_BAZEL_ARGS-} \
--strategy=Javac=worker --worker_quit_after_build \
--fetch --nostamp \
- --javacopt="-source ${JAVA_VERSION} -target ${JAVA_VERSION}" \
${BAZEL_TARGET} || return $?
if [ -n "${BAZEL_SUM}" ]; then
cat bazel-genfiles/src/java.version >${BAZEL_SUM}
@@ -93,7 +103,7 @@
fi
if [ -z "${BOOTSTRAP:-}" ]; then
tempdir
- BOOTSTRAP=${NEW_TMPDIR}/bazel
+ BOOTSTRAP=${NEW_TMPDIR}/bazel${BAZEL_FLAVOUR}
local FILE=bazel-bin/${BAZEL_TARGET##//}
cp -f ${FILE/:/\/} $BOOTSTRAP
chmod +x $BOOTSTRAP
diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh
index abb7e1e..f949dbd 100755
--- a/scripts/bootstrap/compile.sh
+++ b/scripts/bootstrap/compile.sh
@@ -17,7 +17,7 @@
# Script for building bazel from scratch without bazel
PROTO_FILES=$(ls src/main/protobuf/*.proto)
-LIBRARY_JARS=$(find third_party -name '*.jar' | tr "\n" " ")
+LIBRARY_JARS=$(find third_party -name '*.jar' | grep -v "javac.jar" | tr "\n" " ")
DIRS=$(echo src/{java_tools/singlejar/java/com/google/devtools/build/zip,main/java,tools/xcode-common/java/com/google/devtools/build/xcode/{common,util}} ${OUTPUT_DIR}/src)
EXCLUDE_FILES=src/main/java/com/google/devtools/build/lib/server/GrpcServer.java
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
index df7e794..63f7de7 100755
--- a/scripts/ci/build.sh
+++ b/scripts/ci/build.sh
@@ -98,45 +98,6 @@
fi
}
-# Set the various arguments when JDK 7 is required (deprecated).
-# This method is here to continue to build binary release of Bazel
-# for JDK 7. We will drop this method and JDK 7 support when our
-# ci system turn red on this one.
-function setup_jdk7() {
- # This is a JDK 7 JavaBuilder from release 0.1.0.
- local javabuilder_url="https://storage.googleapis.com/bazel/0.1.0/JavaBuilder_deploy.jar"
- local javac_url="https://github.com/bazelbuild/bazel/blob/0.1.0/third_party/java/jdk/langtools/javac.jar?raw=true"
- sed -i.bak 's/_version = "8"/_version = "7"/' tools/jdk/BUILD
- rm -f tools/jdk/BUILD.bak
- rm -f third_party/java/jdk/langtools/javac.jar
- curl -Ls -o tools/jdk/JavaBuilder_deploy.jar "${javabuilder_url}"
- curl -Ls -o third_party/java/jdk/langtools/javac.jar "${javac_url}"
- # Do not use the skylark bootstrapped version of JavaBuilder
- export BAZEL_ARGS="--singlejar_top=//src/java_tools/singlejar:bootstrap_deploy.jar \
- --genclass_top=//src/java_tools/buildjar:bootstrap_genclass_deploy.jar \
- --ijar_top=//third_party/ijar"
- # Skip building JavaBuilder
- export BAZEL_SKIP_TOOL_COMPILATION=tools/jdk/JavaBuilder_deploy.jar
- # Ignore JDK8 tests
- export BAZEL_TEST_FILTERS="-jdk8"
- if ! grep -Fq 'RealJavaBuilder' src/java_tools/buildjar/BUILD; then
- # And more ugly hack. Overwrite the BUILD file of JavaBuilder
- # so we use the pre-built version in integration tests.
- sed -i.bak 's/name = \"JavaBuilder\"/name = \"RealJavaBuilder\"/' \
- src/java_tools/buildjar/BUILD
- rm -f src/java_tools/buildjar/BUILD.bak
- cat >>src/java_tools/buildjar/BUILD <<'EOF'
-genrule(
- name = "JavaBuilder",
- outs = ["JavaBuilder_deploy.jar"],
- srcs = ["//tools/jdk:JavaBuilder_deploy.jar"],
- cmd = "cp $< $@",
- visibility = ["//visibility:public"],
-)
-EOF
- fi
-}
-
# Main entry point for building bazel.
# It sets the embed label to the release name if any, calls the whole
# test suite, compile the various packages, then copy the artifacts
@@ -151,9 +112,10 @@
if [[ "${JAVA_VERSION-}" =~ ^(1\.)?7$ ]]; then
JAVA_VERSION=1.7
- setup_jdk7
+ FLAVOUR="-jdk7"
release_label="${release_label}-jdk7"
else
+ FLAVOUR=""
JAVA_VERSION=1.8
fi
@@ -177,15 +139,16 @@
--workspace_status_command=scripts/ci/build_status_command.sh \
--define JAVA_VERSION=${JAVA_VERSION} \
${ARGS} \
- //scripts/packages/... || exit $?
+ //scripts/packages:install${FLAVOUR}.sh \
+ //scripts/bazel-debian${FLAVOUR}.deb} || exit $?
if [ -n "${1-}" ]; then
# Copy the results to the output directory
mkdir -p $1/packages
cp output/bazel $1/bazel
- cp bazel-bin/scripts/packages/install.sh $1/bazel-${release_label}-installer.sh
+ cp bazel-bin/scripts/packages/install${FLAVOUR}.sh $1/bazel-${release_label}-installer.sh
if [ "$PLATFORM" = "linux" ]; then
- cp bazel-bin/scripts/packages/bazel-debian.deb $1/bazel_${release_label}.deb
+ cp bazel-bin/scripts/packages/bazel-debian${FLAVOUR}.deb $1/bazel_${release_label}.deb
fi
cp bazel-genfiles/scripts/packages/README.md $1/README.md
fi
diff --git a/scripts/packages/BUILD b/scripts/packages/BUILD
index 2fba80f..86ca34d 100644
--- a/scripts/packages/BUILD
+++ b/scripts/packages/BUILD
@@ -33,6 +33,13 @@
)
genrule(
+ name = "rename-bazel-bin-jdk7",
+ srcs = ["//src:bazel_jdk7"],
+ outs = ["jdk7/bazel-real"],
+ cmd = "cp $< $@",
+)
+
+genrule(
name = "rename-bazel-sh",
srcs = ["bazel.sh"],
outs = ["bazel"],
@@ -41,24 +48,49 @@
load(":self_extract_binary.bzl", "self_extract_binary")
-self_extract_binary(
- name = "install.sh",
- empty_files = ["tools/defaults/BUILD"],
- flatten_resources = [
- ":bazel-real",
- ":bazel",
- "//scripts:bash_completion",
- ],
- launcher = ":launcher_bin.sh",
-)
+[
+ self_extract_binary(
+ name = "install%s.sh" % flavour,
+ empty_files = ["tools/defaults/BUILD"],
+ flatten_resources = [
+ ":bazel-real" + flavour,
+ ":bazel",
+ "//scripts:bash_completion",
+ ],
+ launcher = ":launcher_bin.sh",
+ )
+ for flavour in [
+ "",
+ "-jdk7",
+ ]
+]
load("//tools/build_defs/pkg:pkg.bzl", "pkg_tar", "pkg_deb")
pkg_tar(
+ name = "bazel-real-bin",
+ files = [
+ ":bazel-real",
+ ],
+ mode = "0755",
+ package_dir = "/usr/bin",
+ strip_prefix = ".",
+)
+
+pkg_tar(
+ name = "bazel-real-bin-jdk7",
+ files = [
+ ":bazel-real-jdk7",
+ ],
+ mode = "0755",
+ package_dir = "/usr/bin",
+ strip_prefix = "scripts/packages/jdk7/",
+)
+
+pkg_tar(
name = "bazel-bin",
files = [
":bazel",
- ":bazel-real",
],
mode = "0755",
package_dir = "/usr/bin",
@@ -87,63 +119,71 @@
strip_prefix = ".",
)
-pkg_tar(
- name = "debian-data",
- extension = "tar.gz",
- deps = [
- ":bazel-bin",
- ":bazel-completion",
- ":bazel-rc",
- ],
-)
-
-genrule(
- name = "embedded_label",
- srcs = [":debian-data"],
- outs = [
- "version.txt",
- ],
- cmd =
- "find . -name stable-status.txt -exec cat '{}' ';'" +
- " | grep EMBED_LABEL | cut -d ' ' -f 2-" +
- # Debian package manager does not order our rcs correctly, insert a tilde
- # Non RC version number are not matched by the regex, example translations:
- # 0.1.2 0.1.2
- # 0.1.2rc2 0.1.2~rc2
- # 0.1.2rc2-jdk7 0.1.2~rc2-jdk7
- " | sed -E 's/^([0-9\\.]+)([^0-9\\.].*)$$/\\1~\\2/'" +
- " > $@\n",
- stamp = 1,
-)
-
-config_setting(
- name = "jdk7",
- values = {"define": "JAVA_VERSION=1.7"},
-)
-
-pkg_deb(
- name = "bazel-debian",
- architecture = "amd64",
- built_using = "bazel (HEAD)",
- data = ":debian-data",
- depends = select({
- ":jdk7": [
- "java7-jdk | java7-sdk",
+[
+ pkg_tar(
+ name = "debian-data" + flavour,
+ extension = "tar.gz",
+ deps = [
+ ":bazel-real-bin" + flavour,
+ ":bazel-bin",
+ ":bazel-completion",
+ ":bazel-rc",
],
- "//conditions:default": [
- "java8-jdk | java8-sdk",
+ )
+ for flavour in [
+ "",
+ "-jdk7",
+ ]
+]
+
+[
+ genrule(
+ name = "embedded_label" + flavour,
+ srcs = [":debian-data"],
+ outs = [
+ "version%s.txt" % flavour,
],
- }) + [
- "pkg-config",
- "zip",
- "g++",
- "zlib1g-dev",
- "unzip",
- "bash-completion",
- ],
- description_file = "debian/description",
- homepage = "http://bazel.io",
- maintainer = "The Bazel Authors <bazel-dev@googlegroups.com>",
- package = "bazel",
- version_file = ":version.txt",
-)
+ cmd =
+ "find . -name stable-status.txt -exec cat '{}' ';'" +
+ " | grep EMBED_LABEL | cut -d ' ' -f 2-" +
+ # Debian package manager does not order our rcs correctly, insert a tilde
+ # Non RC version number are not matched by the regex, example translations:
+ # 0.1.2 0.1.2
+ # 0.1.2rc2 0.1.2~rc2
+ # 0.1.2rc2-jdk7 0.1.2~rc2-jdk7
+ " | sed -E 's/^([0-9\\.]+)([^0-9\\.].*)$$/\\1~\\2/'" +
+ " > $@\n",
+ stamp = 1,
+ )
+ for flavour in [
+ "",
+ "-jdk7",
+ ]
+]
+
+[
+ pkg_deb(
+ name = "bazel-debian" + flavour,
+ architecture = "amd64",
+ built_using = "bazel (HEAD)",
+ data = ":debian-data" + flavour,
+ depends = [
+ "java7-jdk | java7-sdk" if (flavour == "-jdk7") else "java8-jdk | java8-sdk",
+ "pkg-config",
+ "zip",
+ "g++",
+ "zlib1g-dev",
+ "unzip",
+ "bash-completion",
+ ],
+ description_file = "debian/description",
+ homepage = "http://bazel.io",
+ maintainer = "The Bazel Authors <bazel-dev@googlegroups.com>",
+ package = "bazel",
+ version_file = ":version%s.txt" % flavour,
+ )
+ for flavour in [
+ "",
+ "-jdk7",
+ ]
+]
diff --git a/src/BUILD b/src/BUILD
index 7ff67d4..07cd4fd 100644
--- a/src/BUILD
+++ b/src/BUILD
@@ -38,6 +38,7 @@
}),
) for suffix, embedded_tools in {
"": [":embedded_tools"],
+ "-jdk7": [":embedded_tools-jdk7"],
"_notools": [],
}.items()]
@@ -87,25 +88,32 @@
}),
)
-genrule(
- name = "embedded_tools",
+filegroup(
+ name = "common_embedded_tools",
srcs = [
- ":create_embedded_tools.sh",
- "//tools:embedded_tools_srcs",
- "//third_party:embedded_tools_srcs",
+ "//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass:GenClass_deploy.jar",
"//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/jarhelper:srcs",
+ "//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar",
+ "//src/java_tools/singlejar:SingleJar_deploy.jar",
+ "//src/main/protobuf:srcs",
"//src/tools/android/java/com/google/devtools/build/android:embedded_tools",
"//src/tools/android/java/com/google/devtools/build/android/ideinfo:embedded_tools",
"//src/tools/android/java/com/google/devtools/build/android/idlclass:embedded_tools",
"//src/tools/android/java/com/google/devtools/build/android/incrementaldeployment:srcs",
"//src/tools/android/java/com/google/devtools/build/android/ziputils:embedded_tools",
- "//src/main/protobuf:srcs",
- "//src/java_tools/buildjar:JavaBuilder_deploy.jar",
- "//src/java_tools/singlejar:SingleJar_deploy.jar",
- "//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass:GenClass_deploy.jar",
- "//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar",
+ "//third_party:embedded_tools_srcs",
"//third_party/ijar",
"//third_party/java/apkbuilder:embedded_tools",
+ "//tools:embedded_tools_srcs",
+ ],
+)
+
+genrule(
+ name = "embedded_tools",
+ srcs = [
+ ":create_embedded_tools.sh",
+ ":common_embedded_tools",
+ "//src/java_tools/buildjar:JavaBuilder_deploy.jar",
] + select({
":darwin": [
":darwin_tools",
@@ -122,9 +130,40 @@
cmd = "$(location :create_embedded_tools.sh) $@ $(SRCS)",
)
+genrule(
+ name = "tools_jdk_jdk7_build",
+ srcs = ["//tools/jdk:BUILD"],
+ outs = ["tools_jdk_jdk7.build"],
+ cmd = "cat $< | sed 's|\"toolchain\"|\"toolchain8\"|' | sed 's|\"toolchain7\"|\"toolchain\"|' > $@",
+)
+
+genrule(
+ name = "embedded_tools-jdk7",
+ srcs = [
+ ":create_embedded_tools.sh",
+ ":common_embedded_tools",
+ "@io_bazel_javabuilder_jdk7//file",
+ "@io_bazel_javac_jdk7//file",
+ ":tools_jdk_jdk7_build",
+ ] + select({
+ ":darwin": [
+ ":darwin_tools",
+ "//third_party/ijar:zipper",
+ ],
+ ":darwin_x86_64": [
+ ":darwin_tools",
+ "//third_party/ijar:zipper",
+ ],
+ ":windows": [], # Windows specifically exclude zipper for now.
+ "//conditions:default": ["//third_party/ijar:zipper"],
+ }),
+ outs = ["embedded_tools-jdk7.zip"],
+ cmd = "$(location :create_embedded_tools.sh) $@ $(SRCS)",
+)
+
[genrule(
name = "package-zip" + suffix,
- srcs = ([":embedded_tools.zip"] if embed else []) + [
+ srcs = ([":embedded_%s.zip" % embed] if embed else []) + [
# The script assumes that the embedded tools zip (if exists) is the
# first item here, the deploy jar the second and install base key is the
# third
@@ -144,8 +183,9 @@
cmd = "$(location :package-bazel.sh) $@ " + ("" if embed else "''") + " $(SRCS)",
tools = ["package-bazel.sh"],
) for suffix, embed in [
- ("", True),
- ("_notools", False),
+ ("", "tools"),
+ ("_notools", None),
+ ("-jdk7", "tools-jdk7"),
]]
[genrule(
@@ -166,6 +206,7 @@
) for suffix in [
"",
"_notools",
+ "-jdk7",
]]
# Build an executable named `bazel.exe`.
@@ -186,6 +227,7 @@
) for suffix in [
"",
"_notools",
+ "-jdk7",
]]
config_setting(
diff --git a/src/create_embedded_tools.sh b/src/create_embedded_tools.sh
index e3b751c..78bd9e4 100755
--- a/src/create_embedded_tools.sh
+++ b/src/create_embedded_tools.sh
@@ -28,6 +28,8 @@
for i in $*; do
case "$i" in
+ *javac.jar) OUTPUT_PATH=third_party/java/jdk/langtools/javac.jar ;;
+ *tools_jdk_jdk7.build) OUTPUT_PATH=tools/jdk/BUILD ;;
*JavaBuilder_deploy.jar) OUTPUT_PATH=tools/jdk/JavaBuilder_deploy.jar ;;
*SingleJar_deploy.jar) OUTPUT_PATH=tools/jdk/SingleJar_deploy.jar ;;
*GenClass_deploy.jar) OUTPUT_PATH=tools/jdk/GenClass_deploy.jar ;;
diff --git a/tools/jdk/BUILD b/tools/jdk/BUILD
index 50feeb0..2793124 100644
--- a/tools/jdk/BUILD
+++ b/tools/jdk/BUILD
@@ -102,6 +102,14 @@
target_version = "8",
)
+java_toolchain(
+ name = "toolchain7",
+ encoding = "UTF-8",
+ jvm_opts = ["-client"],
+ source_version = "7",
+ target_version = "7",
+)
+
filegroup(
name = "srcs",
srcs = [
