Add test for pywrapper script This adds some test coverage for the autodetecting Python toolchain (excludes Windows). Also added improved error handling in the script under test. Fixes #7843. RELNOTES: None PiperOrigin-RevId: 245241521
diff --git a/tools/python/BUILD b/tools/python/BUILD index 2bdfb3e..04d9c23 100644 --- a/tools/python/BUILD +++ b/tools/python/BUILD
@@ -1,3 +1,5 @@ +load(":utils.bzl", "expand_pyversion_template") + package(default_visibility = ["//visibility:public"]) filegroup( @@ -22,3 +24,19 @@ ], visibility = ["//tools:__pkg__"], ) + +expand_pyversion_template( + name = "_generate_wrappers", + out2 = ":py2wrapper.sh", + out3 = ":py3wrapper.sh", + template = "pywrapper_template.txt", + visibility = ["//visibility:private"], +) + +py_test( + name = "pywrapper_test", + srcs = ["pywrapper_test.py"], + data = [":py2wrapper.sh"], + python_version = "PY2", + deps = ["//src/test/py/bazel:test_base"], +)
diff --git a/tools/python/pywrapper_template.txt b/tools/python/pywrapper_template.txt index 0b73fa0..be85c69 100644 --- a/tools/python/pywrapper_template.txt +++ b/tools/python/pywrapper_template.txt
@@ -1,18 +1,22 @@ -#!/bin/sh -eu +#!/bin/sh -u +# Don't set -e because we don't have robust trapping and printing of errors. -# Use /bin/sh rather than /bin/bash for portability. See discussion here: +# We use /bin/sh rather than /bin/bash for portability. See discussion here: # https://groups.google.com/forum/?nomobile=true#!topic/bazel-dev/4Ql_7eDcLC0 # We do lose the ability to set -o pipefail. # TODO(#7843): integration tests for failure to find python / wrong version # found / error while trying to print version -GENERAL_FAILURE_MESSAGE="Error: The default python toolchain \ -(@bazel_tools//tools/python:autodetecting_toolchain) was unable to locate a \ -suitable Python interpreter on the target platform at execution time. Please \ -register an appropriate Python toolchain. See the documentation for \ -py_runtime_pair here: -https://github.com/bazelbuild/bazel/blob/master/tools/python/toolchain.bzl." +FAILURE_HEADER="\ +Error occurred while attempting to use the default Python toolchain \ +(@bazel_tools//tools/python:autodetecting_toolchain)." + +die() { + echo "$FAILURE_HEADER" 1>&2 + echo "$1" 1>&2 + exit 1 +} # Try the "python%VERSION%" command name first, then fall back on "python". PYTHON_BIN=`which python%VERSION% || echo ""` @@ -22,20 +26,30 @@ USED_FALLBACK=1 fi if [ -z "${PYTHON_BIN:-}" ]; then - echo "$GENERAL_FAILURE_MESSAGE" - echo "Failure reason: Cannot locate 'python%VERSION%' or 'python' on the \ -target platform's PATH, which is: -$PATH" - exit 1 + die "Neither 'python%VERSION%' nor 'python' were found on the target \ +platform's PATH, which is: + +$PATH + +Please ensure an interpreter is available on this platform (and marked \ +executable), or else register an appropriate Python toolchain as per the \ +documentation for py_runtime_pair \ +(https://github.com/bazelbuild/bazel/blob/master/tools/python/toolchain.bzl)." fi # Verify that we grabbed an interpreter with the right version. -VERSION_STR=`"$PYTHON_BIN" -V 2>&1` +VERSION_STR=`"$PYTHON_BIN" -V 2>&1` \ + || die "Could not get interpreter version via '$PYTHON_BIN -V'" if ! echo "$VERSION_STR" | grep -q " %VERSION%\." ; then - echo "$GENERAL_FAILURE_MESSAGE" - echo "Failure reason: According to '$PYTHON_BIN -V', version is \ -'$VERSION_STR', but we need version %VERSION%" - exit 1 + die "According to '$PYTHON_BIN -V', version is '$VERSION_STR', but we \ +need version %VERSION%. PATH is: + +$PATH + +Please ensure an interpreter with version %VERSION% is available on this \ +platform as 'python%VERSION%' or 'python', or else register an appropriate \ +Python toolchain as per the documentation for py_runtime_pair \ +(https://github.com/bazelbuild/bazel/blob/master/tools/python/toolchain.bzl)." fi exec "$PYTHON_BIN" "$@"
diff --git a/tools/python/pywrapper_test.py b/tools/python/pywrapper_test.py new file mode 100644 index 0000000..fa41239 --- /dev/null +++ b/tools/python/pywrapper_test.py
@@ -0,0 +1,195 @@ +# pylint: disable=g-bad-file-header +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +import subprocess +import textwrap +import unittest + +from src.test.py.bazel import test_base + + +class MockPythonLines(object): + + NORMAL = textwrap.dedent(r"""\ + if [ "$1" = "-V" ]; then + echo "Mock Python 2.xyz!" + else + echo "I am mock Python!" + fi + """).split("\n") + + FAIL = textwrap.dedent(r"""\ + echo "Mock failure!" + exit 1 + """).split("\n") + + WRONG_VERSION = textwrap.dedent(r"""\ + if [ "$1" = "-V" ]; then + echo "Mock Python 3.xyz!" + else + echo "I am mock Python!" + fi + """).split("\n") + + VERSION_ERROR = textwrap.dedent(r"""\ + if [ "$1" = "-V" ]; then + echo "Error!" + exit 1 + else + echo "I am mock Python!" + fi + """).split("\n") + + +# TODO(brandjon): Switch to shutil.which when the test is moved to PY3. +def which(cmd): + """A poor man's approximation of `shutil.which()` or the `which` command. + + Args: + cmd: The command (executable) name to lookup; should not contain path + separators + + Returns: + The absolute path to the first match in PATH, or None if not found. + """ + for p in os.environ["PATH"].split(os.pathsep): + fullpath = os.path.abspath(os.path.join(p, cmd)) + if os.path.exists(fullpath): + return fullpath + return None + + +# TODO(brandjon): Move this test to PY3. Blocked (ironically!) on the fix for +# #4815 being available in the host version of Bazel used to run this test. +class PywrapperTest(test_base.TestBase): + """Unit tests for pywrapper_template.txt. + + These tests are based on the instantiation of the template for Python 2. They + ensure that the wrapper can locate, validate, and launch a Python 2 executable + on PATH. To ensure hermeticity, the tests launch the wrapper with PATH + restricted to the scratch directory. + + Unix only. + """ + + def setup_tool(self, cmd): + """Copies a command from its system location to the test directory.""" + path = which(cmd) + self.assertIsNotNone( + path, msg="Could not locate '%s' command on PATH" % cmd) + self.CopyFile(path, os.path.join("dir", cmd), executable=True) + + def setUp(self): + super(PywrapperTest, self).setUp() + + # Locate script under test. + wrapper_path = self.Rlocation("io_bazel/tools/python/py2wrapper.sh") + self.assertIsNotNone( + wrapper_path, msg="Could not locate py2wrapper.sh in runfiles") + self.wrapper_path = wrapper_path + + # Setup scratch directory with all executables the script depends on. + # + # This is brittle, but we need to make sure we can run the script when only + # the scratch directory is on PATH, so that we can control whether or not + # the python executables exist on PATH. + self.setup_tool("which") + self.setup_tool("echo") + self.setup_tool("grep") + + def run_wrapper(self, title_for_logging=None): + new_env = dict(os.environ) + new_env["PATH"] = self.Path("dir") + proc = subprocess.Popen([self.wrapper_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=self.Path("dir"), + env=new_env) + # TODO(brandjon): Add a timeout arg here when upgraded to PY3. + out, err = proc.communicate() + if title_for_logging is not None: + print(textwrap.dedent("""\ + ---------------- + %s + Exit code: %d + stdout: + %s + stderr: + %s + ---------------- + """) % (title_for_logging, proc.returncode, out, err)) + return proc.returncode, out, err + + def assert_wrapper_success(self, returncode, out, err): + self.assertEqual(returncode, 0, msg="Expected to exit without error") + self.assertEqual( + out, "I am mock Python!\n", msg="stdout was not as expected") + self.assertEqual(err, "", msg="Expected to produce no stderr output") + + def assert_wrapper_failure(self, returncode, out, err, message): + self.assertEqual(returncode, 1, msg="Expected to exit with error code 1") + self.assertRegexpMatches( + err, message, msg="stderr did not contain expected string") + + def test_finds_python2(self): + self.ScratchFile("dir/python2", MockPythonLines.NORMAL, executable=True) + returncode, out, err = self.run_wrapper("test_finds_python2") + self.assert_wrapper_success(returncode, out, err) + + def test_finds_python(self): + self.ScratchFile("dir/python", MockPythonLines.NORMAL, executable=True) + returncode, out, err = self.run_wrapper("test_finds_python") + self.assert_wrapper_success(returncode, out, err) + + def test_prefers_python2(self): + self.ScratchFile("dir/python2", MockPythonLines.NORMAL, executable=True) + self.ScratchFile("dir/python", MockPythonLines.FAIL, executable=True) + returncode, out, err = self.run_wrapper("test_prefers_python2") + self.assert_wrapper_success(returncode, out, err) + + def test_no_interpreter_found(self): + returncode, out, err = self.run_wrapper("test_no_interpreter_found") + self.assert_wrapper_failure(returncode, out, err, + "Neither 'python2' nor 'python' were found") + + def test_wrong_version(self): + self.ScratchFile( + "dir/python2", MockPythonLines.WRONG_VERSION, executable=True) + returncode, out, err = self.run_wrapper("test_wrong_version") + self.assert_wrapper_failure( + returncode, out, err, + "version is 'Mock Python 3.xyz!', but we need version 2") + + def test_error_getting_version(self): + self.ScratchFile( + "dir/python2", MockPythonLines.VERSION_ERROR, executable=True) + returncode, out, err = self.run_wrapper("test_error_getting_version") + self.assert_wrapper_failure(returncode, out, err, + "Could not get interpreter version") + + def test_interpreter_not_executable(self): + self.ScratchFile( + "dir/python2", MockPythonLines.VERSION_ERROR, executable=False) + returncode, out, err = self.run_wrapper("test_interpreter_not_executable") + self.assert_wrapper_failure(returncode, out, err, + "Neither 'python2' nor 'python' were found") + + +if __name__ == "__main__": + unittest.main()