test_base.py: use bazel-runfiles to respect repo mappings
Use rules_python to fetch the bazel-runfiles pip package, we can later migrate other checked-in Python deps to rules_python as well.
On linux and macOS, we are using a hermetic Python toolchain fetched by rules_python, which helped avoid issues with older system installed Python version (<=3.6) on centos7 and ubuntu1804.
On windows, the hermetic Python toolchain doesn't work with `pywin32` pip package, therefore we still use the auto-detected python toolchain.
For bootstrapping test, we need to use the auto-detected python toolchain to avoid downloading Python.
Working towards: https://github.com/bazelbuild/bazel/issues/18957
RELNOTES:
PiperOrigin-RevId: 553731631
Change-Id: I605399067e21653cab707694810de24fd2109c41
diff --git a/.bazelrc b/.bazelrc
index 6cdb680..16c688a 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -27,6 +27,11 @@
build:macos --macos_minimum_os=10.11
+# On Windows, we need pywin32 pip package, which doesn't work with the Python hermetic toolchain.
+# See https://github.com/bazelbuild/rules_python/issues/1356
+# Therefore, use the local detected Python toolchain on Windows.
+build:windows --extra_toolchains=@bazel_tools//tools/python:autodetecting_toolchain
+
# Enable Bzlmod
build:bzlmod --enable_bzlmod
build:bzlmod --check_direct_dependencies=error
diff --git a/MODULE.bazel b/MODULE.bazel
index d7a6482..340a332 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -120,3 +120,11 @@
bazel_dev_deps = use_extension("//:extensions.bzl", "bazel_dev_deps")
use_repo(bazel_dev_deps, "local_bazel_source_list")
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+ hub_name = "bazel_pip_dev_deps",
+ python_version = "3.8",
+ requirements_lock = "//:requirements.txt",
+)
+use_repo(pip, "bazel_pip_dev_deps")
diff --git a/WORKSPACE b/WORKSPACE
index 2aa95e0..735de12 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -182,10 +182,6 @@
patch_cmds_win = EXPORT_WORKSPACE_IN_BUILD_BAZEL_FILE_WIN,
)
-load("@rules_python//python:repositories.bzl", "py_repositories")
-
-py_repositories()
-
dist_http_archive(
name = "zstd-jni",
build_file = "//third_party:zstd-jni/zstd-jni.BUILD",
@@ -560,3 +556,23 @@
load("@maven_android//:defs.bzl", pinned_maven_install_android = "pinned_maven_install")
pinned_maven_install_android()
+
+load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")
+
+py_repositories()
+
+python_register_toolchains(
+ name = "python_3_8",
+ python_version = "3.8",
+)
+
+load("@python_3_8//:defs.bzl", "interpreter")
+
+load("@rules_python//python:pip.bzl", "pip_parse")
+pip_parse(
+ name = "bazel_pip_dev_deps",
+ requirements_lock = "//:requirements.txt",
+ python_interpreter_target = interpreter,
+)
+load("@bazel_pip_dev_deps//:requirements.bzl", "install_deps")
+install_deps()
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..ce27260
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+bazel-runfiles==0.24.0
diff --git a/scripts/bootstrap/bootstrap.sh b/scripts/bootstrap/bootstrap.sh
index 7a0040e..befafd5 100755
--- a/scripts/bootstrap/bootstrap.sh
+++ b/scripts/bootstrap/bootstrap.sh
@@ -37,11 +37,17 @@
--compilation_mode=opt \
--distdir=derived/distdir \
--extra_toolchains=//scripts/bootstrap:all \
+ --extra_toolchains=@bazel_tools//tools/python:autodetecting_toolchain \
+ --override_repository=maven="$(get_cwd)/maven" \
${DIST_BOOTSTRAP_ARGS:-} \
${EXTRA_BAZEL_ARGS:-}"
cp scripts/bootstrap/BUILD.bootstrap scripts/bootstrap/BUILD
+# Remove lines containing 'install_deps' to avoid loading @bazel_pip_dev_deps,
+# which requires fetching the python toolchain.
+sed -i.bak '/install_deps/d' WORKSPACE && rm WORKSPACE.bak
+
if [ -z "${BAZEL-}" ]; then
function _run_bootstrapping_bazel() {
local command=$1
diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh
index db183a6..55d7174 100755
--- a/scripts/bootstrap/compile.sh
+++ b/scripts/bootstrap/compile.sh
@@ -468,7 +468,6 @@
--startup_time=329 --extract_data_time=523 \
--rc_source=/dev/null --isatty=1 \
--build_python_zip \
- --override_repository=maven="$(get_cwd)/maven" \
"${client_env[@]}" \
--client_cwd="$(get_cwd)" \
"${@}"
diff --git a/src/test/py/bazel/BUILD b/src/test/py/bazel/BUILD
index a6b9b7a..4911e92 100644
--- a/src/test/py/bazel/BUILD
+++ b/src/test/py/bazel/BUILD
@@ -1,5 +1,6 @@
load("//:distdir_deps.bzl", "gen_workspace_stanza")
load("@rules_python//python:defs.bzl", "py_library", "py_test")
+load("@bazel_pip_dev_deps//:requirements.bzl", "requirement")
package(default_visibility = ["//visibility:private"])
@@ -35,6 +36,7 @@
"//tools/ctexplain:__pkg__",
"//tools/python:__pkg__",
],
+ deps = [requirement("bazel-runfiles")],
)
gen_workspace_stanza(
diff --git a/src/test/py/bazel/test_base.py b/src/test/py/bazel/test_base.py
index 61bcfff..d9c840f 100644
--- a/src/test/py/bazel/test_base.py
+++ b/src/test/py/bazel/test_base.py
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""Bazel Python integration test framework."""
+
import locale
import os
import shutil
@@ -23,6 +25,7 @@
import sys
import tempfile
import unittest
+import runfiles
class Error(Exception):
@@ -43,6 +46,7 @@
class TestBase(unittest.TestCase):
+ """Bazel Python integration test base."""
_runfiles = None
_temp = None
@@ -103,7 +107,7 @@
def setUp(self):
unittest.TestCase.setUp(self)
if self._runfiles is None:
- self._runfiles = TestBase._LoadRunfiles()
+ self._runfiles = runfiles.Create()
test_tmpdir = TestBase._CreateDirs(TestBase.GetEnv('TEST_TMPDIR'))
self._tests_root = TestBase._CreateDirs(
os.path.join(test_tmpdir, 'tests_root'))
@@ -258,10 +262,7 @@
def Rlocation(self, runfile):
"""Returns the absolute path to a runfile."""
- if TestBase.IsWindows():
- return self._runfiles.get(runfile)
- else:
- return os.path.join(self._runfiles, runfile)
+ return self._runfiles.Rlocation(runfile)
def ScratchDir(self, path):
"""Creates directories under the test's scratch directory.
@@ -570,30 +571,6 @@
return env
@staticmethod
- def _LoadRunfiles():
- """Loads the runfiles manifest from ${TEST_SRCDIR}/MANIFEST.
-
- Only necessary to use on Windows, where runfiles are not symlinked in to the
- runfiles directory, but are written to a MANIFEST file instead.
-
- Returns:
- on Windows: {string: string} dictionary, keys are runfiles-relative paths,
- values are absolute paths that the runfiles entry is mapped to;
- on other platforms: string; value of $TEST_SRCDIR
- """
- test_srcdir = TestBase.GetEnv('TEST_SRCDIR')
- if not TestBase.IsWindows():
- return test_srcdir
-
- result = {}
- with open(os.path.join(test_srcdir, 'MANIFEST'), 'r') as f:
- for l in f:
- tokens = l.strip().split(' ')
- if len(tokens) == 2:
- result[tokens[0]] = tokens[1]
- return result
-
- @staticmethod
def _CreateDirs(path):
if not os.path.exists(path):
os.makedirs(path)
diff --git a/src/test/shell/bazel/bazel_determinism_test.sh b/src/test/shell/bazel/bazel_determinism_test.sh
index 8480d42..eaddf59 100755
--- a/src/test/shell/bazel/bazel_determinism_test.sh
+++ b/src/test/shell/bazel/bazel_determinism_test.sh
@@ -61,10 +61,18 @@
# Set up the maven repository properly.
cp maven/BUILD.vendor maven/BUILD
+ # Remove lines containing 'install_deps' to avoid loading @bazel_pip_dev_deps,
+ # which requires fetching the python toolchain.
+ sed -i.bak '/install_deps/d' WORKSPACE && rm WORKSPACE.bak
+
+ # Use @bazel_tools//tools/python:autodetecting_toolchain to avoid
+ # downloading python toolchain.
+
# Build Bazel once.
bazel \
--output_base="${TEST_TMPDIR}/out1" \
build \
+ --extra_toolchains=@bazel_tools//tools/python:autodetecting_toolchain \
--distdir=$distdir \
--override_repository=maven=$maven \
--nostamp \
@@ -77,6 +85,7 @@
--install_base="${TEST_TMPDIR}/install_base2" \
--output_base="${TEST_TMPDIR}/out2" \
build \
+ --extra_toolchains=@bazel_tools//tools/python:autodetecting_toolchain \
--distdir=$distdir \
--override_repository=maven=$maven \
--nostamp \
