Reduce unnecessary concurrent github curls (#2865)
Nightly image builds are failing because the curl command to get the
latest build tool versions are 404ing. This commit attempts to fix that
issue by consolidating all of the separate calls into one call in
build.sh.
diff --git a/buildkite/docker/build.sh b/buildkite/docker/build.sh
index 2af06fa..2c21416 100755
--- a/buildkite/docker/build.sh
+++ b/buildkite/docker/build.sh
@@ -34,6 +34,43 @@
docker buildx create --driver=docker-container --use --name mp-builder
fi
+# Downloads the build tools once per run and bakes them into the images via a
+# named build context. Downloading them in each Dockerfile instead meant
+# excessive concurrent requests to GitHub, which would flakily 404.
+TOOLS_DIR="$(mktemp -d)"
+trap 'rm -rf "$TOOLS_DIR"' EXIT
+
+function latest_release_tag() {
+ local repo="$1"
+ local tag
+ tag=$(curl -sSI "https://github.com/${repo}/releases/latest" | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//')
+ if [[ -z "$tag" ]]; then
+ echo "ERROR: could not determine the latest release of ${repo}" >&2
+ return 1
+ fi
+ echo "$tag"
+}
+
+# Every image is built for at most these architectures, so fetch both and let
+# each Dockerfile pick the one matching the image it is building.
+function download_tool() {
+ local repo="$1"
+ local tool="$2"
+ local tag
+ tag=$(latest_release_tag "$repo")
+ echo "Using ${tool} ${tag}"
+ local arch
+ for arch in amd64 arm64; do
+ curl -fsSLo "${TOOLS_DIR}/${tool}-linux-${arch}" \
+ "https://github.com/${repo}/releases/download/${tag}/${tool}-linux-${arch}"
+ done
+}
+
+download_tool bazelbuild/bazelisk bazelisk
+download_tool bazel-contrib/buildtools buildifier
+
+TOOLS_CONTEXT=(--build-context "tools=$TOOLS_DIR")
+
# A function to build a docker image and only show logs (with --progress=plain) on failure.
# This helps reduce the log size from 500MB to a few KB on success while still surfacing critical errors.
function docker_build() {
@@ -60,28 +97,28 @@
# Containers used by Bazel
# For Rocky Linux & Ubuntu we build multi-platform images.
pids=()
-docker_build -f rockylinux8/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target rockylinux8 -t "gcr.io/$PREFIX/rockylinux8" rockylinux8 & pids+=($!)
+docker_build "${TOOLS_CONTEXT[@]}" -f rockylinux8/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target rockylinux8 -t "gcr.io/$PREFIX/rockylinux8" rockylinux8 & pids+=($!)
docker_build -f debian10/Dockerfile --target debian10-java11 -t "gcr.io/$PREFIX/debian10-java11" debian10 & pids+=($!)
-docker_build -f debian12/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target debian12-java17 -t "gcr.io/$PREFIX/debian12" debian12 & pids+=($!)
-docker_build -f debian13/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target debian13-java21 -t "gcr.io/$PREFIX/debian13" debian13 & pids+=($!)
-docker_build -f ubuntu2204/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2204-java17 -t "gcr.io/$PREFIX/ubuntu2204-java17" ubuntu2204 & pids+=($!)
-docker_build -f ubuntu2204/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2204 -t "gcr.io/$PREFIX/ubuntu2204" ubuntu2204 & pids+=($!)
-docker_build -f ubuntu2404/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2404 -t "gcr.io/$PREFIX/ubuntu2404" ubuntu2404 & pids+=($!)
-docker_build -f fedora39/Dockerfile --target fedora39-java17 -t "gcr.io/$PREFIX/fedora39-java17" fedora39 & pids+=($!)
-docker_build -f fedora40/Dockerfile --target fedora40-java21 -t "gcr.io/$PREFIX/fedora40-java21" fedora40 & pids+=($!)
-docker_build -f fedora43/Dockerfile --target fedora43-java25 -t "gcr.io/$PREFIX/fedora43-java25" fedora43 & pids+=($!)
+docker_build "${TOOLS_CONTEXT[@]}" -f debian12/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target debian12-java17 -t "gcr.io/$PREFIX/debian12" debian12 & pids+=($!)
+docker_build "${TOOLS_CONTEXT[@]}" -f debian13/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target debian13-java21 -t "gcr.io/$PREFIX/debian13" debian13 & pids+=($!)
+docker_build "${TOOLS_CONTEXT[@]}" -f ubuntu2204/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2204-java17 -t "gcr.io/$PREFIX/ubuntu2204-java17" ubuntu2204 & pids+=($!)
+docker_build "${TOOLS_CONTEXT[@]}" -f ubuntu2204/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2204 -t "gcr.io/$PREFIX/ubuntu2204" ubuntu2204 & pids+=($!)
+docker_build "${TOOLS_CONTEXT[@]}" -f ubuntu2404/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2404 -t "gcr.io/$PREFIX/ubuntu2404" ubuntu2404 & pids+=($!)
+docker_build "${TOOLS_CONTEXT[@]}" -f fedora39/Dockerfile --target fedora39-java17 -t "gcr.io/$PREFIX/fedora39-java17" fedora39 & pids+=($!)
+docker_build "${TOOLS_CONTEXT[@]}" -f fedora40/Dockerfile --target fedora40-java21 -t "gcr.io/$PREFIX/fedora40-java21" fedora40 & pids+=($!)
+docker_build "${TOOLS_CONTEXT[@]}" -f fedora43/Dockerfile --target fedora43-java25 -t "gcr.io/$PREFIX/fedora43-java25" fedora43 & pids+=($!)
for pid in "${pids[@]}"; do
wait "$pid"
done
-docker_build -f rockylinux8/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target rockylinux8-java8 -t "gcr.io/$PREFIX/rockylinux8-java8" rockylinux8
-docker_build -f rockylinux8/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target rockylinux8-java11 -t "gcr.io/$PREFIX/rockylinux8-java11" rockylinux8
-docker_build -f rockylinux8/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target rockylinux8-java11-devtoolset10 -t "gcr.io/$PREFIX/rockylinux8-java11-devtoolset10" rockylinux8
-docker_build -f rockylinux8/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target rockylinux8-releaser -t "gcr.io/$PREFIX/rockylinux8-releaser" rockylinux8
-docker_build -f ubuntu2204/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2204-kythe -t "gcr.io/$PREFIX/ubuntu2204-kythe" ubuntu2204
-docker_build -f ubuntu2204/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2204-bazel-java17 -t "gcr.io/$PREFIX/ubuntu2204-bazel-java17" ubuntu2204
-docker_build -f ubuntu2404/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2404-kythe -t "gcr.io/$PREFIX/ubuntu2404-kythe" ubuntu2404
-docker_build -f fedora39/Dockerfile --target fedora39-bazel-java17 -t "gcr.io/$PREFIX/fedora39-bazel-java17" fedora39
-docker_build -f fedora40/Dockerfile --target fedora40-bazel-java21 -t "gcr.io/$PREFIX/fedora40-bazel-java21" fedora40
-docker_build -f fedora43/Dockerfile --target fedora43-bazel-java25 -t "gcr.io/$PREFIX/fedora43-bazel-java25" fedora43
+docker_build "${TOOLS_CONTEXT[@]}" -f rockylinux8/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target rockylinux8-java8 -t "gcr.io/$PREFIX/rockylinux8-java8" rockylinux8
+docker_build "${TOOLS_CONTEXT[@]}" -f rockylinux8/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target rockylinux8-java11 -t "gcr.io/$PREFIX/rockylinux8-java11" rockylinux8
+docker_build "${TOOLS_CONTEXT[@]}" -f rockylinux8/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target rockylinux8-java11-devtoolset10 -t "gcr.io/$PREFIX/rockylinux8-java11-devtoolset10" rockylinux8
+docker_build "${TOOLS_CONTEXT[@]}" -f rockylinux8/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target rockylinux8-releaser -t "gcr.io/$PREFIX/rockylinux8-releaser" rockylinux8
+docker_build "${TOOLS_CONTEXT[@]}" -f ubuntu2204/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2204-kythe -t "gcr.io/$PREFIX/ubuntu2204-kythe" ubuntu2204
+docker_build "${TOOLS_CONTEXT[@]}" -f ubuntu2204/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2204-bazel-java17 -t "gcr.io/$PREFIX/ubuntu2204-bazel-java17" ubuntu2204
+docker_build "${TOOLS_CONTEXT[@]}" -f ubuntu2404/Dockerfile --builder mp-builder --load --platform=linux/amd64,linux/arm64 --target ubuntu2404-kythe -t "gcr.io/$PREFIX/ubuntu2404-kythe" ubuntu2404
+docker_build "${TOOLS_CONTEXT[@]}" -f fedora39/Dockerfile --target fedora39-bazel-java17 -t "gcr.io/$PREFIX/fedora39-bazel-java17" fedora39
+docker_build "${TOOLS_CONTEXT[@]}" -f fedora40/Dockerfile --target fedora40-bazel-java21 -t "gcr.io/$PREFIX/fedora40-bazel-java21" fedora40
+docker_build "${TOOLS_CONTEXT[@]}" -f fedora43/Dockerfile --target fedora43-bazel-java25 -t "gcr.io/$PREFIX/fedora43-bazel-java25" fedora43
diff --git a/buildkite/docker/debian12/Dockerfile b/buildkite/docker/debian12/Dockerfile
index 834d81c..429594f 100644
--- a/buildkite/docker/debian12/Dockerfile
+++ b/buildkite/docker/debian12/Dockerfile
@@ -71,13 +71,7 @@
rm -rf /var/lib/apt/lists/*
# Bazelisk
-RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \
- chown root:root /usr/local/bin/bazel && \
- chmod 0755 /usr/local/bin/bazel
+COPY --from=tools --chmod=0755 bazelisk-linux-${TARGETARCH} /usr/local/bin/bazel
# Buildifier
-RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazel-contrib/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/buildifier https://github.com/bazel-contrib/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \
- chown root:root /usr/local/bin/buildifier && \
- chmod 0755 /usr/local/bin/buildifier
+COPY --from=tools --chmod=0755 buildifier-linux-${TARGETARCH} /usr/local/bin/buildifier
diff --git a/buildkite/docker/debian13/Dockerfile b/buildkite/docker/debian13/Dockerfile
index 7ac843b..17cc0d9 100644
--- a/buildkite/docker/debian13/Dockerfile
+++ b/buildkite/docker/debian13/Dockerfile
@@ -71,13 +71,7 @@
rm -rf /var/lib/apt/lists/*
# Bazelisk
-RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \
- chown root:root /usr/local/bin/bazel && \
- chmod 0755 /usr/local/bin/bazel
+COPY --from=tools --chmod=0755 bazelisk-linux-${TARGETARCH} /usr/local/bin/bazel
# Buildifier
-RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazel-contrib/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/buildifier https://github.com/bazel-contrib/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \
- chown root:root /usr/local/bin/buildifier && \
- chmod 0755 /usr/local/bin/buildifier
+COPY --from=tools --chmod=0755 buildifier-linux-${TARGETARCH} /usr/local/bin/buildifier
diff --git a/buildkite/docker/fedora39/Dockerfile b/buildkite/docker/fedora39/Dockerfile
index c9a3521..9fa6159 100644
--- a/buildkite/docker/fedora39/Dockerfile
+++ b/buildkite/docker/fedora39/Dockerfile
@@ -54,14 +54,8 @@
google-cloud-cli
# Bazelisk
-RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \
- chown root:root /usr/local/bin/bazel && \
- chmod 0755 /usr/local/bin/bazel
+COPY --from=tools --chmod=0755 bazelisk-linux-${BUILDARCH} /usr/local/bin/bazel
# Buildifier
-RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazel-contrib/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/buildifier https://github.com/bazel-contrib/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \
- chown root:root /usr/local/bin/buildifier && \
- chmod 0755 /usr/local/bin/buildifier
+COPY --from=tools --chmod=0755 buildifier-linux-${BUILDARCH} /usr/local/bin/buildifier
diff --git a/buildkite/docker/fedora40/Dockerfile b/buildkite/docker/fedora40/Dockerfile
index 35ffd30..6af4082 100644
--- a/buildkite/docker/fedora40/Dockerfile
+++ b/buildkite/docker/fedora40/Dockerfile
@@ -54,13 +54,7 @@
google-cloud-cli
# Bazelisk
-RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \
- chown root:root /usr/local/bin/bazel && \
- chmod 0755 /usr/local/bin/bazel
+COPY --from=tools --chmod=0755 bazelisk-linux-${BUILDARCH} /usr/local/bin/bazel
# Buildifier
-RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazel-contrib/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/buildifier https://github.com/bazel-contrib/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \
- chown root:root /usr/local/bin/buildifier && \
- chmod 0755 /usr/local/bin/buildifier
+COPY --from=tools --chmod=0755 buildifier-linux-${BUILDARCH} /usr/local/bin/buildifier
diff --git a/buildkite/docker/fedora43/Dockerfile b/buildkite/docker/fedora43/Dockerfile
index 3f225c3..e5c02a5 100644
--- a/buildkite/docker/fedora43/Dockerfile
+++ b/buildkite/docker/fedora43/Dockerfile
@@ -55,13 +55,7 @@
&& rm -rf /var/cache/yum
# Bazelisk
-RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -fLo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${BUILDARCH} && \
- chown root:root /usr/local/bin/bazel && \
- chmod 0755 /usr/local/bin/bazel
+COPY --from=tools --chmod=0755 bazelisk-linux-${BUILDARCH} /usr/local/bin/bazel
# Buildifier
-RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazel-contrib/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -fLo /usr/local/bin/buildifier https://github.com/bazel-contrib/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${BUILDARCH} && \
- chown root:root /usr/local/bin/buildifier && \
- chmod 0755 /usr/local/bin/buildifier
+COPY --from=tools --chmod=0755 buildifier-linux-${BUILDARCH} /usr/local/bin/buildifier
diff --git a/buildkite/docker/rockylinux8/Dockerfile b/buildkite/docker/rockylinux8/Dockerfile
index c493f1e..b2c1efa 100644
--- a/buildkite/docker/rockylinux8/Dockerfile
+++ b/buildkite/docker/rockylinux8/Dockerfile
@@ -50,16 +50,10 @@
RUN localedef -i en_US -f ISO-8859-1 en_US.ISO-8859-1
# Bazelisk
-RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \
- chown root:root /usr/local/bin/bazel && \
- chmod 0755 /usr/local/bin/bazel
+COPY --from=tools --chmod=0755 bazelisk-linux-${TARGETARCH} /usr/local/bin/bazel
# Buildifier
-RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazel-contrib/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/buildifier https://github.com/bazel-contrib/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \
- chown root:root /usr/local/bin/buildifier && \
- chmod 0755 /usr/local/bin/buildifier
+COPY --from=tools --chmod=0755 buildifier-linux-${TARGETARCH} /usr/local/bin/buildifier
FROM rockylinux8-nojdk AS rockylinux8-nojdk-devtoolset10
diff --git a/buildkite/docker/ubuntu2204/Dockerfile b/buildkite/docker/ubuntu2204/Dockerfile
index 01c1241..04ee473 100644
--- a/buildkite/docker/ubuntu2204/Dockerfile
+++ b/buildkite/docker/ubuntu2204/Dockerfile
@@ -73,16 +73,10 @@
rm -rf /var/lib/apt/lists/*
# Bazelisk
-RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \
- chown root:root /usr/local/bin/bazel && \
- chmod 0755 /usr/local/bin/bazel
+COPY --from=tools --chmod=0755 bazelisk-linux-${TARGETARCH} /usr/local/bin/bazel
# Buildifier
-RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazel-contrib/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/buildifier https://github.com/bazel-contrib/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \
- chown root:root /usr/local/bin/buildifier && \
- chmod 0755 /usr/local/bin/buildifier
+COPY --from=tools --chmod=0755 buildifier-linux-${TARGETARCH} /usr/local/bin/buildifier
FROM ubuntu2204-bazel-nojdk AS ubuntu2204-bazel-java17
diff --git a/buildkite/docker/ubuntu2404/Dockerfile b/buildkite/docker/ubuntu2404/Dockerfile
index 19511a6..2499802 100644
--- a/buildkite/docker/ubuntu2404/Dockerfile
+++ b/buildkite/docker/ubuntu2404/Dockerfile
@@ -73,16 +73,10 @@
rm -rf /var/lib/apt/lists/*
# Bazelisk
-RUN LATEST_BAZELISK=$(curl -sSI https://github.com/bazelbuild/bazelisk/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/${LATEST_BAZELISK}/bazelisk-linux-${TARGETARCH} && \
- chown root:root /usr/local/bin/bazel && \
- chmod 0755 /usr/local/bin/bazel
+COPY --from=tools --chmod=0755 bazelisk-linux-${TARGETARCH} /usr/local/bin/bazel
# Buildifier
-RUN LATEST_BUILDIFIER=$(curl -sSI https://github.com/bazel-contrib/buildtools/releases/latest | grep -i '^location: ' | sed 's|.*/||' | sed $'s/\r//') && \
- curl -Lo /usr/local/bin/buildifier https://github.com/bazel-contrib/buildtools/releases/download/${LATEST_BUILDIFIER}/buildifier-linux-${TARGETARCH} && \
- chown root:root /usr/local/bin/buildifier && \
- chmod 0755 /usr/local/bin/buildifier
+COPY --from=tools --chmod=0755 buildifier-linux-${TARGETARCH} /usr/local/bin/buildifier
FROM ubuntu2404-nojdk AS ubuntu2404
