Implement autodetecting Python toolchain
This replaces the stub default Python toolchain with one that actually locates the target platform's Python interpreter at runtime. Try it out with
bazel build //some_py_binary --experimental_use_python_toolchains
and note that, unlike before (#4815), the correct Python interpreter gets invoked by default regardless of whether you specify `--python_version=PY2` or `--python_version=PY3`.
This toolchain is only intended as a last resort, if the user doesn't define and register a better toolchain (that satisfies the target platform constraints).
Work toward #7375 and #4815. Follow-up work needed to add a test (#7843) and windows support (#7844).
RELNOTES: None
PiperOrigin-RevId: 240417315
diff --git a/tools/python/BUILD.tools b/tools/python/BUILD.tools
index 1efd754..35c789b 100644
--- a/tools/python/BUILD.tools
+++ b/tools/python/BUILD.tools
@@ -1,5 +1,5 @@
load(":python_version.bzl", "define_python_version_flag")
-load(":toolchain.bzl", "py_runtime_pair")
+load(":toolchain.bzl", "define_autodetecting_toolchain")
package(default_visibility = ["//visibility:public"])
@@ -65,17 +65,14 @@
# system Python 3 interpreter on a platform.
constraint_setting(name = "py3_interpreter_path")
-# A Python toolchain that, at execution time, attempts to detect a platform
-# runtime having the appropriate major Python version.
+# Definitions for a Python toolchain that, at execution time, attempts to detect
+# a platform runtime having the appropriate major Python version.
+#
+# This is a toolchain of last resort that gets automatically registered in all
+# workspaces. Ideally you should register your own Python toolchain, which will
+# supersede this one so long as its constraints match the target platform.
-py_runtime_pair(
- name = "autodetecting_py_runtime_pair",
- # TODO(brandjon): Not yet implemented. Currently this provides no runtimes,
- # so it will just fail at analysis time if you attempt to use it.
-)
-
-toolchain(
+define_autodetecting_toolchain(
name = "autodetecting_toolchain",
- toolchain = ":autodetecting_py_runtime_pair",
- toolchain_type = ":toolchain_type",
+ pywrapper_template = "pywrapper_template.txt",
)
diff --git a/tools/python/python_version.bzl b/tools/python/python_version.bzl
index 8bb45b8..aecad07 100644
--- a/tools/python/python_version.bzl
+++ b/tools/python/python_version.bzl
@@ -50,12 +50,19 @@
def define_python_version_flag(name):
"""Defines the target to expose the Python version to select().
- For use only by @bazel_tools//python:BUILD; see the documentation comment
- there.
+ For use only by @bazel_tools//tools/python:BUILD; see the documentation
+ comment there.
Args:
- name: The name of the target to introduce.
+ name: The name of the target to introduce. Must have value
+ "python_version". This param is present only to make the BUILD file
+ more readable.
"""
+ if native.package_name() != "tools/python":
+ fail("define_python_version_flag() is private to " +
+ "@bazel_tools//tools/python")
+ if name != "python_version":
+ fail("Python version flag must be named 'python_version'")
# Config settings for the underlying native flags we depend on:
# --force_python, --python_version, and --incompatible_py3_is_default.
diff --git a/tools/python/pywrapper_template.txt b/tools/python/pywrapper_template.txt
new file mode 100644
index 0000000..c2f5b18
--- /dev/null
+++ b/tools/python/pywrapper_template.txt
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# TODO(#7843): integration tests for failure to find python / wrong version
+# found / error while trying to print version
+
+set -euo pipefail
+
+GENERAL_FAILURE_MESSAGE="Error: The default python toolchain \
+(@bazel_tools//tools/python:autodetecting_toolchain) was unable to locate a \
+suitable Python interpreter on the target platform at execution time. Please \
+register an appropriate Python toolchain. See the documentation for \
+py_runtime_pair here:
+https://github.com/bazelbuild/bazel/blob/master/tools/python/toolchain.bzl."
+
+# Try the "python%VERSION%" command name first, then fall back on "python".
+PYTHON_BIN=$(which python%VERSION% || echo "")
+USED_FALLBACK=0
+if [[ -z "${PYTHON_BIN:-}" ]]; then
+ PYTHON_BIN=$(which python || echo "")
+ USED_FALLBACK=1
+fi
+if [[ -z "${PYTHON_BIN:-}" ]]; then
+ echo "$GENERAL_FAILURE_MESSAGE"
+ echo "Failure reason: Cannot locate 'python%VERSION%' or 'python' on the \
+target platform's PATH, which is:
+$PATH"
+fi
+
+# Verify that we grabbed an interpreter with the right version.
+VERSION_STR=$("$PYTHON_BIN" -V 2>&1)
+if ! grep -q " %VERSION%\." <<< $VERSION_STR; then
+ echo "$GENERAL_FAILURE_MESSAGE"
+ echo "Failure reason: According to '$PYTHON_BIN -V', version is \
+'$VERSION_STR', but we need version %VERSION%"
+fi
+
+exec "$PYTHON_BIN" "$@"
diff --git a/tools/python/toolchain.bzl b/tools/python/toolchain.bzl
index 3fe9475c..656ab62 100644
--- a/tools/python/toolchain.bzl
+++ b/tools/python/toolchain.bzl
@@ -14,6 +14,8 @@
"""Definitions related to the Python toolchain."""
+load(":utils.bzl", "expand_pyversion_template")
+
def _py_runtime_pair_impl(ctx):
if ctx.attr.py2_runtime != None:
py2_runtime = ctx.attr.py2_runtime[PyRuntimeInfo]
@@ -62,7 +64,7 @@
This rule returns a `platform_common.ToolchainInfo` provider with the following
schema:
-```
+```python
platform_common.ToolchainInfo(
py2_runtime = <PyRuntimeInfo or None>,
py3_runtime = <PyRuntimeInfo or None>,
@@ -71,7 +73,9 @@
Example usage:
-```
+```python
+# In your BUILD file...
+
load("@bazel_tools//tools/python/toolchain.bzl", "py_runtime_pair")
py_runtime(
@@ -99,5 +103,76 @@
toolchain_type = "@bazel_tools//tools/python:toolchain_type",
)
```
+
+```python
+# In your WORKSPACE...
+
+register_toolchains("//my_pkg:my_toolchain")
+```
""",
)
+
+# TODO(#7844): Add support for a windows (.bat) version of the autodetecting
+# toolchain, based on the "py" wrapper (e.g. "py -2" and "py -3"). Use select()
+# in the template attr of the _generate_*wrapper targets.
+
+def define_autodetecting_toolchain(name, pywrapper_template):
+ """Defines the autodetecting Python toolchain.
+
+ For use only by @bazel_tools//tools/python:BUILD; see the documentation
+ comment there.
+
+ Args:
+ name: The name of the toolchain to introduce. Must have value
+ "autodetecting_toolchain". This param is present only to make the
+ BUILD file more readable.
+ pywrapper_template: The label of the pywrapper_template.txt file.
+ """
+ if native.package_name() != "tools/python":
+ fail("define_autodetecting_toolchain() is private to " +
+ "@bazel_tools//tools/python")
+ if name != "autodetecting_toolchain":
+ fail("Python autodetecting toolchain must be named " +
+ "'autodetecting_toolchain'")
+
+ expand_pyversion_template(
+ name = "_generate_wrappers",
+ template = pywrapper_template,
+ out2 = ":py2wrapper.sh",
+ out3 = ":py3wrapper.sh",
+ visibility = ["//visibility:private"],
+ )
+
+ # Note that the pywrapper script is a .sh file, not a sh_binary target. If
+ # we needed to make it a proper shell target, e.g. because it needed to
+ # access runfiles and needed to depend on the runfiles library, then we'd
+ # have to use a workaround to allow it to be depended on by py_runtime. See
+ # https://github.com/bazelbuild/bazel/issues/4286#issuecomment-475661317.
+
+ native.py_runtime(
+ name = "_autodetecting_py2_runtime",
+ interpreter = ":py2wrapper.sh",
+ python_version = "PY2",
+ visibility = ["//visibility:private"],
+ )
+
+ native.py_runtime(
+ name = "_autodetecting_py3_runtime",
+ interpreter = ":py3wrapper.sh",
+ python_version = "PY3",
+ visibility = ["//visibility:private"],
+ )
+
+ py_runtime_pair(
+ name = "_autodetecting_py_runtime_pair",
+ py2_runtime = ":_autodetecting_py2_runtime",
+ py3_runtime = ":_autodetecting_py3_runtime",
+ visibility = ["//visibility:public"],
+ )
+
+ native.toolchain(
+ name = name,
+ toolchain = ":_autodetecting_py_runtime_pair",
+ toolchain_type = ":toolchain_type",
+ visibility = ["//visibility:public"],
+ )
diff --git a/tools/python/utils.bzl b/tools/python/utils.bzl
new file mode 100644
index 0000000..9278e32
--- /dev/null
+++ b/tools/python/utils.bzl
@@ -0,0 +1,53 @@
+# Copyright 2019 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for the @bazel_tools//tools/python package.
+
+This file does not access any Python-rules-specific logic, and is therefore
+less likely to be broken by Python-related changes. That in turn means this
+file is less likely to cause bootstrapping issues.
+"""
+
+def _expand_pyversion_template_impl(ctx):
+ if ctx.outputs.out2:
+ ctx.actions.expand_template(
+ template = ctx.file.template,
+ output = ctx.outputs.out2,
+ substitutions = {"%VERSION%": "2"},
+ is_executable = True,
+ )
+ if ctx.outputs.out3:
+ ctx.actions.expand_template(
+ template = ctx.file.template,
+ output = ctx.outputs.out3,
+ substitutions = {"%VERSION%": "3"},
+ is_executable = True,
+ )
+
+expand_pyversion_template = rule(
+ implementation = _expand_pyversion_template_impl,
+ attrs = {
+ "template": attr.label(
+ allow_single_file = True,
+ doc = "The input template file.",
+ ),
+ "out2": attr.output(doc = """\
+The output file produced by substituting "%VERSION%" with "2"."""),
+ "out3": attr.output(doc = """\
+The output file produced by substituting "%VERSION%" with "3"."""),
+ },
+ doc = """\
+Given a template file, generates two expansions by replacing the substring
+"%VERSION%" with "2" and "3".""",
+)
