Implement docs generation pipeline on Buildkite (#1114)
* Implement docs generation pipeline on Buildkite
After this change we can replace our CloudBuild pipelines with Buildkite pipelines. They'll run as GitHub postsubmit and create and upload new versions of the Bazel documentation, the blog and the website.
Moreover, this code will be used by a "docs staging" presubmit pipeline, which still needs some work, though.
Next steps:
1. Create Buildkite pipelines
2. Disable CloudBuild pipelines
3. Enable Buildkite pipelines
4. Update (internal) documentation
5. Fix and enable staging pipeline
Progress towards #1110 (Milestone 1)
diff --git a/docgen/Dockerfile b/docgen/Dockerfile
new file mode 100644
index 0000000..f1ec9dd
--- /dev/null
+++ b/docgen/Dockerfile
@@ -0,0 +1,42 @@
+# Based on:
+# bazelbuild/bazel/scripts/docs/Dockerfile
+# bazel-blog/scripts/Dockerfile
+# bazelbuild/continuous-integration/buildkite/docker/ubuntu1804/Dockerfile
+
+FROM ubuntu:18.04
+
+ENV DEBIAN_FRONTEND="noninteractive"
+RUN apt-get -qqy update && \
+ apt-get -qqy install build-essential curl liblzma-dev \
+ python3.7 python-pygments ruby ruby-dev unzip zlib1g-dev && \
+ apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN ln -fs /usr/bin/python3.7 /usr/bin/python
+
+### Install packages required by bazelci.py
+RUN dpkg --add-architecture i386 && \
+ apt-get -qqy update && \
+ apt-get -qqy install --no-install-recommends \
+ python3-requests \
+ python3-yaml \
+ && \
+ apt-get -qqy purge apport && \
+ rm -rf /var/lib/apt/lists/*
+
+### Install Bazelisk.
+RUN curl -fLo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.7.5/bazelisk-linux-amd64 && \
+ chown root:root /usr/local/bin/bazel && \
+ chmod 0755 /usr/local/bin/bazel
+RUN bazel version
+
+### Install Google Cloud SDK.
+### https://cloud.google.com/sdk/docs/quickstart-debian-ubuntu
+RUN export CLOUD_SDK_REPO="cloud-sdk-bionic" && \
+ echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \
+ curl -L https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
+ apt-get -qqy update && \
+ apt-get -qqy install google-cloud-sdk && \
+ rm -rf /var/lib/apt/lists/*
+
+COPY Gemfile .
+RUN gem install -g --no-rdoc --no-ri && rm -f Gemfile
diff --git a/docgen/Gemfile b/docgen/Gemfile
new file mode 100644
index 0000000..6512903
--- /dev/null
+++ b/docgen/Gemfile
@@ -0,0 +1,6 @@
+gem 'jekyll', '~> 3.8.6'
+gem 'jekyll-paginate', '~> 1.0'
+gem 'pygments.rb', '~> 0.6.0'
+gem 'redcarpet', '~> 3.2', '>= 3.2.3'
+gem 'jekyll-toc', '~> 0.13.1'
+gem 'jekyll-sitemap', '~> 1.4.0'
diff --git a/docgen/build.sh b/docgen/build.sh
new file mode 100755
index 0000000..189bf71
--- /dev/null
+++ b/docgen/build.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# gcloud builds submit -t gcr.io/bazel-public/docgen .
+
+docker build -t docgen .
+docker tag docgen gcr.io/bazel-public/docgen
+docker push gcr.io/bazel-public/docgen
diff --git a/docgen/docgen.py b/docgen/docgen.py
new file mode 100644
index 0000000..9d03b4a
--- /dev/null
+++ b/docgen/docgen.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+#
+# Copyright 2021 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import os
+import subprocess
+import sys
+
+import bazelci
+
+DEFAULT_FLAGS = ["--action_env=PATH=/usr/local/bin:/usr/bin:/bin"]
+
+Settings = collections.namedtuple(
+ "Settings", ["target", "build_flags", "output_dir", "gcs_bucket", "gcs_subdir", "landing_page"]
+)
+
+DOCGEN_SETTINGS = {
+ "bazel-trusted": {
+ "https://github.com/bazelbuild/bazel.git": Settings(
+ target="//site",
+ build_flags=[],
+ output_dir="bazel-bin/site/site-build",
+ gcs_bucket="docs.bazel.build",
+ gcs_subdir="",
+ landing_page="versions/master/bazel-overview.html",
+ ),
+ "https://github.com/bazelbuild/bazel-blog.git": Settings(
+ target="//:site",
+ build_flags=[],
+ output_dir="bazel-bin/site-build",
+ gcs_bucket="blog.bazel.build",
+ gcs_subdir="",
+ landing_page="index.html",
+ ),
+ "https://github.com/bazelbuild/bazel-website.git": Settings(
+ target="//:site",
+ build_flags=[],
+ output_dir="bazel-bin/site-build",
+ gcs_bucket="www.bazel.build",
+ gcs_subdir="",
+ landing_page="index.html",
+ ),
+ },
+}
+
+
+def get_destination(bucket, subdir):
+ if not subdir:
+ return bucket
+
+ return "{}/{}".format(bucket, subdir)
+
+
+def get_url(dest_bucket, landing_page):
+ return "{}/{}".format(
+ dest_bucket.replace("gs://", "https://storage.googleapis.com/"), landing_page
+ )
+
+
+def main(argv=None):
+ org = os.getenv("BUILDKITE_ORGANIZATION_SLUG")
+ repo = os.getenv("BUILDKITE_REPO")
+ settings = DOCGEN_SETTINGS.get(org, {}).get(repo)
+ if not settings:
+ bazelci.eprint("docgen is not enabled for '%s' org and repository %s", org, repo)
+ return 1
+
+ bazelci.print_expanded_group(":bazel: Building documentation from {}".format(repo))
+ try:
+ bazelci.execute_command(
+ ["bazel", "build"] + DEFAULT_FLAGS + settings.build_flags + [settings.target]
+ )
+ except subprocess.CalledProcessError as e:
+ bazelci.eprint("Bazel failed with exit code {}".format(e.returncode))
+ return e.returncode
+
+ bucket = "gs://{}".format(settings.gcs_bucket)
+ dest = get_destination(bucket, settings.gcs_subdir)
+ bazelci.print_expanded_group(":bazel: Uploading documentation to {}".format(dest))
+ try:
+ bazelci.execute_command(
+ ["gsutil", "-m", "rsync", "-r", "-c", "-d", settings.output_dir, dest]
+ )
+ bazelci.execute_command(
+ ["gsutil", "web", "set", "-m", "index.html", "-e", "404.html", bucket]
+ )
+ # TODO: does not work with 404 pages in sub directories
+ except subprocess.CalledProcessError as e:
+ bazelci.eprint("Upload to GCS failed with exit code {}".format(e.returncode))
+ return e.returncode
+
+ bazelci.print_collapsed_group(":bazel: Publishing documentation URL")
+ message = "You can find the documentation at {}".format(get_url(dest, settings.landing_page))
+ bazelci.execute_command(
+ ["buildkite-agent", "annotate", "--style=info", message, "--context", "doc_url"]
+ )
+
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/pipelines/bazel-docgen.yml b/pipelines/bazel-docgen.yml
new file mode 100644
index 0000000..eba9d38
--- /dev/null
+++ b/pipelines/bazel-docgen.yml
@@ -0,0 +1,28 @@
+---
+steps:
+- label: DocGen
+ command:
+ - curl -sS https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/buildkite/bazelci.py?$(date +%s) -o bazelci.py
+ - curl -sS https://raw.githubusercontent.com/bazelbuild/continuous-integration/master/docgen/docgen.py?$(date +%s) -o docgen.py
+ - python3.6 docgen.py
+ agents:
+ queue: default
+ plugins:
+ docker#v3.5.0:
+ image: gcr.io/bazel-public/docgen
+ network: host
+ volumes:
+ - "/etc/group:/etc/group:ro"
+ - "/etc/passwd:/etc/passwd:ro"
+ - "/opt:/opt:ro"
+ - "/var/lib/buildkite-agent:/var/lib/buildkite-agent"
+ - "/var/lib/gitmirrors:/var/lib/gitmirrors:ro"
+ - "/var/run/docker.sock:/var/run/docker.sock"
+ privileged: true
+ always-pull: true
+ environment:
+ - ANDROID_HOME
+ - ANDROID_NDK_HOME
+ - BUILDKITE_ARTIFACT_UPLOAD_DESTINATION
+ propagate-uid-gid: true
+ propagate-environment: true
