Make Bazel tests compatible with Starlark implementation of Python rules
Summary of changes:
* Remove usage of Python rules where not necessary; the Python rules
were/are being used to test features that aren't specific to the Python
rules.
* Use regex matching for failures instead of exact strings. The Starlark
rules have slightly different phrasing or orderings of elements that
don't affect behavior. This also makes the tests less brittle overall.
* Disable attempting to use the Starlark implementation in all the tests
requiring Python 2 (e.g. srcs_version checking etc). The Starlark
implementation doesn't currently (and probably never will) support Python
2. https://github.com/bazelbuild/bazel/issues/15684 filed to track removal
of it in Bazel.
* Disable attempting to use the Starlark implementation where toolchain
resolution is required. This will eventually be implemented, but isn't
is use at Google and otherwise blocks switching internally.
* Tests verifying warnings are printed were removed; Starlark doesn't
provide a warning facility.
* Stamping is disabled in various tests because, with the Starlark implementation,
it requires remote execution of actions, which some tests aren't setup for
(they never needed it previously). Rather than set this up, stamping was
disabled (affected tests don't require stamping anyways).
PiperOrigin-RevId: 455695015
Change-Id: I82822eff05b6c0a66e65f131c9e1c8784b1573ac
diff --git a/src/test/shell/integration/aquery_test.sh b/src/test/shell/integration/aquery_test.sh
index eee4a15..ba6f553 100755
--- a/src/test/shell/integration/aquery_test.sh
+++ b/src/test/shell/integration/aquery_test.sh
@@ -1434,7 +1434,6 @@
|| fail "Expected success"
cat output >> "$TEST_log"
- assert_contains "PYTHON_BINARY = '%python_binary%'" output
assert_contains "{%python_binary%:" output
bazel aquery --output=jsonproto ${QUERY} > output 2> "$TEST_log" \
diff --git a/src/test/shell/integration/modify_execution_info_test.sh b/src/test/shell/integration/modify_execution_info_test.sh
index 28e485e..80c1d46 100755
--- a/src/test/shell/integration/modify_execution_info_test.sh
+++ b/src/test/shell/integration/modify_execution_info_test.sh
@@ -241,7 +241,7 @@
Turbine=+requires-turbine,\
JavaSourceJar=+requires-java-source-jar,\
Javac=+requires-javac,\
-PyTinypar=+requires-py-tinypar,\
+Py.*=+requires-py,\
Action=+requires-action \
> output 2> "$TEST_log" || fail "Expected success"
@@ -260,9 +260,7 @@
assert_contains "requires-java-source-jar: ''" output
assert_contains "requires-proto: ''" output # GenProtoDescriptorSet should match
if [[ "$PRODUCT_NAME" != "bazel" ]]; then
- # Python rules generate some cpp actions and local actions, but py-tinypar
- # is the main unique-to-python rule which runs remotely for a py_binary.
- assert_contains "requires-py-tinypar: ''" output
+ assert_contains "requires-py: ''" output
fi
}
diff --git a/src/test/shell/integration/python_stub_test.sh b/src/test/shell/integration/python_stub_test.sh
index 734e20a..42e16bed 100755
--- a/src/test/shell/integration/python_stub_test.sh
+++ b/src/test/shell/integration/python_stub_test.sh
@@ -83,10 +83,18 @@
)
EOF
- bazel run //test:main2 \
- &> $TEST_log || fail "bazel run failed"
- expect_log "I am Python 2"
- bazel run //test:main3 \
+ # Google builds don't support Python 2
+ if [[ "$PRODUCT_NAME" == "bazel" ]]; then
+ bazel run //test:main2 \
+ &> $TEST_log || fail "bazel run failed"
+ expect_log "I am Python 2"
+ fi
+
+ # Stamping is disabled so that the invocation doesn't time out. What
+ # happens is Google has stamping enabled by default, which causes the
+ # Starlark rule implementation to run an action, which then tries to run
+ # remotely, but network access is disabled by default, so it times out.
+ bazel run --nostamp //test:main3 \
&> $TEST_log || fail "bazel run failed"
expect_log "I am Python 3"
}
@@ -117,6 +125,10 @@
# to a value different from the top-level configuration, and then changing it
# back again, is able to reuse the top-level configuration.
function test_no_action_conflicts_from_version_transition() {
+ # Requires Python 2 support, which doesn't work for Google-internal builds
+ if [[ "$PRODUCT_NAME" != "bazel" ]]; then
+ return 0
+ fi
mkdir -p test
# To repro, we need to build a C++ target in two different ways in the same
diff --git a/src/test/shell/integration/python_test.sh b/src/test/shell/integration/python_test.sh
index a8538be..34b19b3 100755
--- a/src/test/shell/integration/python_test.sh
+++ b/src/test/shell/integration/python_test.sh
@@ -73,6 +73,14 @@
# - Otherwise, put your test in //src/test/shell/bazel. That suite can invoke
# actual Python 2 and 3 interpreters.
+# Python's import system distinguishes between "regular files" and other files
+# in some cases (e.g. /dev/null). Here, we are trying to verify that a generated
+# __init__.py file passes the isfile() check that the import logic performs and
+# would fail for a "character device" (e.g. symlinked to /dev/null). For more
+# info, see:
+# * https://github.com/bazelbuild/bazel/issues/1458
+# * https://github.com/bazelbuild/bazel/issues/2394
+# * https://bugs.python.org/issue28425
function test_python_binary_empty_files_in_runfiles_are_regular_files() {
mkdir -p test/mypackage
cat > test/BUILD <<'EOF'
@@ -101,11 +109,11 @@
print("mypackage/__init__.py does not exist")
sys.exit(1)
-if os.path.islink(file_to_check):
- print("mypackage/__init__.py is a symlink, expected a regular file")
- sys.exit(1)
+# Symlinks to regular files are OK.
+realpath = os.path.realpath(file_to_check)
+print("{} realpath is: {}".format(file_to_check, realpath))
-if not os.path.isfile(file_to_check):
+if not os.path.isfile(realpath):
print("mypackage/__init__.py is not a regular file")
sys.exit(1)
@@ -132,10 +140,14 @@
data = [':py-tool'],
)
EOF
- bazel build --experimental_build_transitive_python_runfiles :sh-tool
+ # Stamping is disabled so that the invocation doesn't time out. What
+ # happens is Google has stamping enabled by default, which causes the
+ # Starlark rule implementation to run an action, which then tries to run
+ # remotely, but network access is disabled by default, so it times out.
+ bazel build --experimental_build_transitive_python_runfiles --nostamp :sh-tool
[ -d "bazel-bin/py-tool${EXE_EXT}.runfiles" ] || fail "py_binary runfiles tree not built"
bazel clean
- bazel build --noexperimental_build_transitive_python_runfiles :sh-tool
+ bazel build --noexperimental_build_transitive_python_runfiles --nostamp :sh-tool
[ ! -e "bazel-bin/py-tool${EXE_EXT}.runfiles" ] || fail "py_binary runfiles tree built"
}
diff --git a/src/test/shell/integration/runfiles_test.sh b/src/test/shell/integration/runfiles_test.sh
index 16f2ec6..8138d61 100755
--- a/src/test/shell/integration/runfiles_test.sh
+++ b/src/test/shell/integration/runfiles_test.sh
@@ -77,7 +77,7 @@
cd $pkg
mkdir -p a/b c/d e/f/g x/y
- touch py.py a/b/no_module.py c/d/one_module.py c/__init__.py e/f/g/ignored.py x/y/z.sh
+ touch py.py a/b/no_module.py c/d/one_module.py c/__init__.py e/f/g/ignored.txt x/y/z.sh
chmod +x x/y/z.sh
cd ..
@@ -99,13 +99,13 @@
if [[ -f "$path" ]]; then
effective_type=file
elif [[ -d "$path" ]]; then
- effective_type=dir
+ effective_type="$actual_type dir"
else
# The various special file types shouldn't occur in practice, so just
# call them unknown
effective_type=unknown
fi
- echo "$path $actual_type $effective_type"
+ echo "$path $effective_type"
done
}
@@ -113,20 +113,37 @@
function test_hidden() {
local -r pkg=$FUNCNAME
- create_pkg $pkg
+
+ mkdir -p "$pkg/e/f/g"
+ touch "$pkg/e/f/g/hidden.txt"
+ cat > "$pkg/defs.bzl" << EOF
+def _obscured_impl(ctx):
+ executable = ctx.actions.declare_file(ctx.label.name)
+ ctx.actions.write(executable, "# nop")
+ return [DefaultInfo(
+ executable = executable,
+ runfiles = ctx.runfiles(files = ctx.files.data),
+ )]
+
+obscured = rule(
+ implementation = _obscured_impl,
+ attrs = {"data": attr.label_list(allow_files = True)},
+ # Must be executable to trigger the obscured runfile check
+ executable = True,
+)
+EOF
+
cat > $pkg/BUILD << EOF
-py_binary(name = "py",
- srcs = [ "py.py" ],
- data = [ "e/f",
- "e/f/g/hidden.py" ])
+load(":defs.bzl", "obscured")
+obscured(name="bin", data=["e/f", "e/f/g/hidden.txt"])
genrule(name = "hidden",
- outs = [ "e/f/g/hidden.py" ],
+ outs = [ "e/f/g/hidden.txt" ],
cmd = "touch \$@")
EOF
- bazel build $pkg:py $EXTRA_BUILD_FLAGS >&$TEST_log 2>&1 || fail "build failed"
+ bazel build $pkg:bin $EXTRA_BUILD_FLAGS >&$TEST_log 2>&1 || fail "build failed"
- # we get a warning that hidden.py is inaccessible
- expect_log_once "${pkg}/e/f/g/hidden.py obscured by ${pkg}/e/f "
+ # we get a warning that hidden.txt is inaccessible
+ expect_log_once "${pkg}/e/f/g/hidden.txt obscured by ${pkg}/e/f "
}
function test_foo_runfiles() {
@@ -147,7 +164,8 @@
"a/b/no_module.py",
"c/d/one_module.py",
"c/__init__.py",
- "e/f/g/ignored.py" ],
+ ],
+ data = ["e/f/g/ignored.txt"],
deps = ["//:root"])
EOF
bazel build $pkg:foo $EXTRA_BUILD_FLAGS >&$TEST_log || fail "build failed"
@@ -174,30 +192,30 @@
test \! -s __init__.py
cd ..
- # These are 3-tuples of (path actualFileType effectiveFileType),
+ # These are basically tuples of (path filetype)
expected="
. regular dir
-./__init__.py symlink file
+./__init__.py file
./test_foo_runfiles regular dir
-./test_foo_runfiles/__init__.py regular file
+./test_foo_runfiles/__init__.py file
./test_foo_runfiles/a regular dir
-./test_foo_runfiles/a/__init__.py regular file
+./test_foo_runfiles/a/__init__.py file
./test_foo_runfiles/a/b regular dir
-./test_foo_runfiles/a/b/__init__.py regular file
-./test_foo_runfiles/a/b/no_module.py symlink file
+./test_foo_runfiles/a/b/__init__.py file
+./test_foo_runfiles/a/b/no_module.py file
./test_foo_runfiles/c regular dir
-./test_foo_runfiles/c/__init__.py symlink file
+./test_foo_runfiles/c/__init__.py file
./test_foo_runfiles/c/d regular dir
-./test_foo_runfiles/c/d/__init__.py regular file
-./test_foo_runfiles/c/d/one_module.py symlink file
+./test_foo_runfiles/c/d/__init__.py file
+./test_foo_runfiles/c/d/one_module.py file
./test_foo_runfiles/e regular dir
./test_foo_runfiles/e/f symlink dir
-./test_foo_runfiles/foo symlink file
-./test_foo_runfiles/py symlink file
-./test_foo_runfiles/py.py symlink file
+./test_foo_runfiles/foo file
+./test_foo_runfiles/py file
+./test_foo_runfiles/py.py file
./test_foo_runfiles/x regular dir
./test_foo_runfiles/x/y regular dir
-./test_foo_runfiles/x/y/z.sh symlink file
+./test_foo_runfiles/x/y/z.sh file
"
expected="$expected$(get_python_runtime_runfiles)"
@@ -205,8 +223,8 @@
# but on Linux we only build `bin`.
if "$is_windows"; then
expected="${expected}
-./test_foo_runfiles/py.exe symlink file
-./test_foo_runfiles/foo.exe symlink file
+./test_foo_runfiles/py.exe file
+./test_foo_runfiles/foo.exe file
"
fi
