src/test/shell/bazel/python_version_test.sh

#!/bin/bash
#
# Copyright 2018 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Test Python 2/3 version behavior. These tests require that the target platform
# has both Python versions available.

# --- begin runfiles.bash initialization ---
# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash).
set -euo pipefail
if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
  if [[ -f "$0.runfiles_manifest" ]]; then
    export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
  elif [[ -f "$0.runfiles/MANIFEST" ]]; then
    export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
  elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
    export RUNFILES_DIR="$0.runfiles"
  fi
fi
if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
  source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
  source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
            "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
else
  echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
  exit 1
fi
# --- end runfiles.bash initialization ---

source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \
  || { echo "integration_test_setup.sh not found!" >&2; exit 1; }

# `uname` returns the current platform, e.g "MSYS_NT-10.0" or "Linux".
# `tr` converts all upper case letters to lower case.
# `case` matches the result if the `uname | tr` expression to string prefixes
# that use the same wildcards as names do in Bash, i.e. "msys*" matches strings
# starting with "msys", and "*" matches everything (it's the default case).
case "$(uname -s | tr [:upper:] [:lower:])" in
msys*)
  # As of 2018-08-14, Bazel on Windows only supports MSYS Bash.
  declare -r is_windows=true
  # As of 2018-12-17, this test is disabled on windows (via "no_windows" tag),
  # so this code shouldn't even have run. See the TODO at
  # use_system_python_2_3_runtimes.
  fail "This test does not run on Windows."
  ;;
darwin*)
  # As of 2018-12-17, this test is disabled on mac, but there's no "no_mac" tag
  # so we just have to trivially succeed. See the TODO at
  # use_system_python_2_3_runtimes.
  echo "This test does not run on Mac; exiting early." >&2
  exit 0
  ;;
*)
  declare -r is_windows=false
  ;;
esac

if "$is_windows"; then
  # Disable MSYS path conversion that converts path-looking command arguments to
  # Windows paths (even if they arguments are not in fact paths).
  export MSYS_NO_PATHCONV=1
  export MSYS2_ARG_CONV_EXCL="*"
fi

# Use a py_runtime that invokes either the system's Python 2 or Python 3
# interpreter based on the Python mode. On Unix this is a workaround for #4815.
#
# TODO(brandjon): Get this running on windows by creating .bat wrappers that
# invoke "py -2" and "py -3". Make sure our windows workers have both Python
# versions installed.
#
# TODO(brandjon): Get this running on mac -- our workers lack a Python 2
# installation.
#
function use_system_python_2_3_runtimes() {
  PYTHON2_BIN=$(which python2 || echo "")
  PYTHON3_BIN=$(which python3 || echo "")
  # Debug output.
  echo "Python 2 interpreter: ${PYTHON2_BIN:-"Not found"}"
  echo "Python 3 interpreter: ${PYTHON3_BIN:-"Not found"}"
  # Fail if either isn't present.
  if [[ -z "${PYTHON2_BIN:-}" || -z "${PYTHON3_BIN:-}" ]]; then
    fail "Can't use system interpreter: Could not find one or both of \
'python2', 'python3'"
  fi

  # Point Python builds at a py_runtime target defined in a //tools package of
  # the main repo. This is not related to @bazel_tools//tools/python.
  add_to_bazelrc "build --python_top=//tools/python:default_runtime"

  mkdir -p tools/python

  cat > tools/python/BUILD << EOF
package(default_visibility=["//visibility:public"])

py_runtime(
    name = "default_runtime",
    files = [],
    interpreter_path = select({
        "@bazel_tools//tools/python:PY2": "${PYTHON2_BIN}",
        "@bazel_tools//tools/python:PY3": "${PYTHON3_BIN}",
    }),
)
EOF
}

function set_up() {
  use_system_python_2_3_runtimes
}

#### TESTS #############################################################

# Sanity test that our environment setup above works.
function test_can_run_py_binaries() {
  mkdir -p test

  cat > test/BUILD << EOF
py_binary(
    name = "main2",
    default_python_version = "PY2",
    srcs = ['main2.py'],
)
py_binary(
    name = "main3",
    default_python_version = "PY3",
    srcs = ["main3.py"],
)
EOF

  cat > test/main2.py << EOF
import platform
print("I am Python " + platform.python_version_tuple()[0])
EOF
  cp test/main2.py test/main3.py
  chmod u+x test/main2.py test/main3.py

  bazel run //test:main2 \
      &> $TEST_log || fail "bazel run failed"
  expect_log "I am Python 2"

  bazel run //test:main3 \
      &> $TEST_log || fail "bazel run failed"
  expect_log "I am Python 3"
}

# Test that access to runfiles works (in general, and under our test environment
# specifically).
function test_can_access_runfiles() {
  mkdir -p test

  cat > test/BUILD << EOF
py_binary(
  name = "main",
  srcs = ["main.py"],
  deps = ["@bazel_tools//tools/python/runfiles"],
  data = ["data.txt"],
)
EOF

  cat > test/data.txt << EOF
abcdefg
EOF

  cat > test/main.py << EOF
from bazel_tools.tools.python.runfiles import runfiles

r = runfiles.Create()
path = r.Rlocation("$WORKSPACE_NAME/test/data.txt")
print("Rlocation returned: " + str(path))
if path is not None:
  with open(path, 'rt') as f:
    print("File contents: " + f.read())
EOF
  chmod u+x test/main.py

  bazel build //test:main || fail "bazel build failed"
  MAIN_BIN=$(bazel info bazel-bin)/test/main
  RUNFILES_MANIFEST_FILE= RUNFILES_DIR= $MAIN_BIN &> $TEST_log
  expect_log "File contents: abcdefg"
}

function test_pybin_can_have_different_version_pybin_as_data_dep() {
  mkdir -p test

  cat > test/BUILD << EOF
py_binary(
  name = "py2bin",
  srcs = ["py2bin.py"],
  default_python_version = "PY2",
)
py_binary(
  name = "py3bin",
  srcs = ["py3bin.py"],
  default_python_version = "PY3",
)
py_binary(
  name = "py2bin_calling_py3bin",
  srcs = ["py2bin_calling_py3bin.py"],
  deps = ["@bazel_tools//tools/python/runfiles"],
  data = [":py3bin"],
  default_python_version = "PY2",
)
py_binary(
  name = "py3bin_calling_py2bin",
  srcs = ["py3bin_calling_py2bin.py"],
  deps = ["@bazel_tools//tools/python/runfiles"],
  data = [":py2bin"],
  default_python_version = "PY3",
)
EOF

  cat > test/py2bin.py << EOF
import platform

print("Inner bin uses Python " + platform.python_version_tuple()[0])
EOF
  chmod u+x test/py2bin.py
  cp test/py2bin.py test/py3bin.py

  cat > test/py2bin_calling_py3bin.py << EOF
import platform
import subprocess
from bazel_tools.tools.python.runfiles import runfiles

r = runfiles.Create()
bin_path = r.Rlocation("$WORKSPACE_NAME/test/py3bin")

print("Outer bin uses Python " + platform.python_version_tuple()[0])
subprocess.call([bin_path])
EOF
  sed s/py3bin/py2bin/ test/py2bin_calling_py3bin.py > test/py3bin_calling_py2bin.py
  chmod u+x test/py2bin_calling_py3bin.py test/py3bin_calling_py2bin.py

  EXPFLAG="--incompatible_allow_python_version_transitions=true"

  bazel build $EXPFLAG //test:py2bin_calling_py3bin //test:py3bin_calling_py2bin \
      || fail "bazel build failed"
  PY2_OUTER_BIN=$(bazel info bazel-bin $EXPFLAG)/test/py2bin_calling_py3bin
  PY3_OUTER_BIN=$(bazel info bazel-bin $EXPFLAG --force_python=PY3)/test/py3bin_calling_py2bin

  RUNFILES_MANIFEST_FILE= RUNFILES_DIR= $PY2_OUTER_BIN &> $TEST_log
  expect_log "Outer bin uses Python 2"
  expect_log "Inner bin uses Python 3"

  RUNFILES_MANIFEST_FILE= RUNFILES_DIR= $PY3_OUTER_BIN &> $TEST_log
  expect_log "Outer bin uses Python 3"
  expect_log "Inner bin uses Python 2"
}

function test_shbin_can_have_different_version_pybins_as_data_deps() {
  mkdir -p test

  cat > test/BUILD << EOF
py_binary(
  name = "py2bin",
  srcs = ["py2bin.py"],
  default_python_version = "PY2",
)
py_binary(
  name = "py3bin",
  srcs = ["py3bin.py"],
  default_python_version = "PY3",
)
sh_binary(
  name = "shbin_calling_py23bins",
  srcs = ["shbin_calling_py23bins.sh"],
  deps = ["@bazel_tools//tools/bash/runfiles"],
  data = [":py2bin", ":py3bin"],
)
EOF

  cat > test/py2bin.py << EOF
import platform

print("py2bin uses Python " + platform.python_version_tuple()[0])
EOF
  sed s/py2bin/py3bin/ test/py2bin.py > test/py3bin.py
  chmod u+x test/py2bin.py test/py3bin.py

  # The workspace name is initialized in testenv.sh; use that var rather than
  # hardcoding it here. The extra sed pass is so we can selectively expand that
  # one var while keeping the rest of the heredoc literal.
  cat | sed "s/{{WORKSPACE_NAME}}/$WORKSPACE_NAME/" > test/shbin_calling_py23bins.sh << 'EOF'
# --- begin runfiles.bash initialization ---
# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash).
set -euo pipefail
if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
  if [[ -f "$0.runfiles_manifest" ]]; then
    export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
  elif [[ -f "$0.runfiles/MANIFEST" ]]; then
    export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
  elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
    export RUNFILES_DIR="$0.runfiles"
  fi
fi
if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
  source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
  source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
            "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
else
  echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
  exit 1
fi
# --- end runfiles.bash initialization ---

$(rlocation {{WORKSPACE_NAME}}/test/py2bin)
$(rlocation {{WORKSPACE_NAME}}/test/py3bin)
EOF

  chmod u+x test/shbin_calling_py23bins.sh

  EXPFLAG="--incompatible_allow_python_version_transitions=true"

  bazel build $EXPFLAG //test:shbin_calling_py23bins \
      || fail "bazel build failed"
  SH_BIN=$(bazel info bazel-bin $EXPFLAG)/test/shbin_calling_py23bins

  RUNFILES_MANIFEST_FILE= RUNFILES_DIR= $SH_BIN &> $TEST_log
  expect_log "py2bin uses Python 2"
  expect_log "py3bin uses Python 3"
}

function test_genrule_can_have_different_version_pybins_as_tools() {
  # This test currently checks that we can use --host_force_python to get
  # PY2 and PY3 binaries in tools. In the future we'll support both modes in the
  # same build without a flag (#6443).

  mkdir -p test

  cat > test/BUILD << 'EOF'
py_binary(
  name = "pybin",
  srcs = ["pybin.py"],
)
genrule(
  name = "genrule_calling_pybin",
  outs = ["out.txt"],
  tools = [":pybin"],
  cmd = "$(location :pybin) > $(location out.txt)"
)
EOF

  cat > test/pybin.py << EOF
import platform

print("pybin uses Python " + platform.python_version_tuple()[0])
EOF
  chmod u+x test/pybin.py

  # Run under both old and new semantics.
  for EXPFLAG in \
      "--incompatible_allow_python_version_transitions=true" \
      "--incompatible_allow_python_version_transitions=false"; do
    echo "Using $EXPFLAG" > $TEST_log
    bazel build $EXPFLAG --host_force_python=PY2 //test:genrule_calling_pybin \
        || fail "bazel build failed"
    ARTIFACT=$(bazel info bazel-genfiles $EXPFLAG)/test/out.txt
    cat $ARTIFACT > $TEST_log
    expect_log "pybin uses Python 2"

    echo "Using $EXPFLAG" > $TEST_log
    bazel build $EXPFLAG --host_force_python=PY3 //test:genrule_calling_pybin \
          || fail "bazel build failed"
      ARTIFACT=$(bazel info bazel-genfiles $EXPFLAG)/test/out.txt
      cat $ARTIFACT > $TEST_log
      expect_log "pybin uses Python 3"
  done
}

function test_can_build_same_target_for_both_versions_in_one_build() {
  mkdir -p test

  cat > test/BUILD << EOF
py_library(
  name = "common",
  srcs = ["common.py"],
)
py_binary(
  name = "py2bin",
  srcs = ["py2bin.py"],
  deps = [":common"],
  default_python_version = "PY2",
)
py_binary(
  name = "py3bin",
  srcs = ["py3bin.py"],
  deps = [":common"],
  default_python_version = "PY3",
)
sh_binary(
  name = "shbin",
  srcs = ["shbin.sh"],
  deps = ["@bazel_tools//tools/bash/runfiles"],
  data = [":py2bin", ":py3bin"],
)
EOF

  cat > test/common.py << EOF
import platform

print("common using Python " + platform.python_version_tuple()[0])
EOF

  cat > test/py2bin.py << EOF
import common
EOF
  chmod u+x test/py2bin.py
  cp test/py2bin.py test/py3bin.py

  # The workspace name is initialized in testenv.sh; use that var rather than
  # hardcoding it here. The extra sed pass is so we can selectively expand that
  # one var while keeping the rest of the heredoc literal.
  cat | sed "s/{{WORKSPACE_NAME}}/$WORKSPACE_NAME/" > test/shbin.sh << 'EOF'
# --- begin runfiles.bash initialization ---
# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash).
set -euo pipefail
if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
  if [[ -f "$0.runfiles_manifest" ]]; then
    export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
  elif [[ -f "$0.runfiles/MANIFEST" ]]; then
    export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
  elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
    export RUNFILES_DIR="$0.runfiles"
  fi
fi
if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
  source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
  source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
            "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
else
  echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
  exit 1
fi
# --- end runfiles.bash initialization ---

$(rlocation {{WORKSPACE_NAME}}/test/py2bin)
$(rlocation {{WORKSPACE_NAME}}/test/py3bin)
EOF
  chmod u+x test/shbin.sh

  EXPFLAG="--incompatible_allow_python_version_transitions=true"

  bazel build $EXPFLAG //test:shbin \
      || fail "bazel build failed"
  SH_BIN=$(bazel info bazel-bin)/test/shbin

  RUNFILES_MANIFEST_FILE= RUNFILES_DIR= $SH_BIN &> $TEST_log
  expect_log "common using Python 2"
  expect_log "common using Python 3"
}

# TODO(brandjon): Rename this file to python_test.sh or else move the below to
# a separate suite.

# Tests that a non-standard library module on the PYTHONPATH added by Bazel
# can override the standard library. This behavior is not necessarily ideal, but
# it is the current semantics; see #6532 about changing that.
function test_source_file_does_not_override_standard_library() {
  mkdir -p test

  cat > test/BUILD << EOF
py_binary(
    name = "main",
    srcs = ["main.py"],
    deps = [":lib"],
    # Pass the empty string, to include the path to this package (within
    # runfiles) on the PYTHONPATH.
    imports = [""],
)

py_library(
    name = "lib",
    # A src name that clashes with a standard library module, such that this
    # local file can take precedence over the standard one depending on its
    # order in PYTHONPATH. Not just any module name would work. For instance,
    # "import sys" gets the built-in module regardless of whether there's some
    # "sys.py" file on the PYTHONPATH. This is probably because built-in modules
    # (i.e., those implemented in C) use a different loader than
    # Python-implemented ones, even though they're both part of the standard
    # distribution of the interpreter.
    srcs = ["re.py"],
)
EOF
  cat > test/main.py << EOF
import re
EOF
  cat > test/re.py << EOF
print("I am lib!")
EOF

  bazel run //test:main \
      &> $TEST_log || fail "bazel run failed"
  # Indicates that the local module overrode the system one.
  expect_log "I am lib!"
}

run_suite "Tests for how the Python rules handle Python 2 vs Python 3"