Google Git
Sign in
bazel / bazel / ca95fecde07a28736ea815ec64bcd639a234d79c / . / src / test / shell / bazel / remote / remote_execution_test.sh
blob: d9c09f9da6e65cfaf915d5bef5f01775481d13db [file] [log] [blame]
#!/bin/bash
#
# Copyright 2016 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Tests remote execution and caching.
set -euo pipefail
# --- begin runfiles.bash initialization ---
if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
if [[ -f "$0.runfiles_manifest" ]]; then
export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
elif [[ -f "$0.runfiles/MANIFEST" ]]; then
export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
export RUNFILES_DIR="$0.runfiles"
fi
fi
if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
"$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
else
echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
exit 1
fi
# --- end runfiles.bash initialization ---
source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \
|| { echo "integration_test_setup.sh not found!" >&2; exit 1; }
source "$(rlocation "io_bazel/src/test/shell/bazel/remote/remote_utils.sh")" \
|| { echo "remote_utils.sh not found!" >&2; exit 1; }
function set_up() {
start_worker \
--incompatible_remote_symlinks
}
function tear_down() {
bazel clean >& $TEST_log
stop_worker
}
case "$(uname -s | tr [:upper:] [:lower:])" in
msys*|mingw*|cygwin*)
declare -r is_windows=true
;;
*)
declare -r is_windows=false
;;
esac
if "$is_windows"; then
export MSYS_NO_PATHCONV=1
export MSYS2_ARG_CONV_EXCL="*"
declare -r EXE_EXT=".exe"
else
declare -r EXE_EXT=""
fi
function has_utf8_locale() {
charmap="$(LC_ALL=en_US.UTF-8 locale charmap 2>/dev/null)"
[[ "${charmap}" == "UTF-8" ]]
}
function setup_credential_helper() {
cat > "${TEST_TMPDIR}/credhelper" <<'EOF'
#!/usr/bin/env python3
print("""{"headers":{"Authorization":["Bearer secret_token"]}}""")
EOF
chmod +x "${TEST_TMPDIR}/credhelper"
}
function test_credential_helper_remote_cache() {
setup_credential_helper
mkdir -p a
cat > a/BUILD <<'EOF'
genrule(
name = "a",
outs = ["a.txt"],
cmd = "touch $(OUTS)",
)
EOF
stop_worker
start_worker --expected_authorization_token=secret_token
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//a:a >& $TEST_log && fail "Build without credentials should have failed"
expect_log "Failed to query remote execution capabilities"
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--experimental_credential_helper="${TEST_TMPDIR}/credhelper" \
//a:a >& $TEST_log || fail "Build with credentials should have succeeded"
}
function test_credential_helper_remote_execution() {
setup_credential_helper
mkdir -p a
cat > a/BUILD <<'EOF'
genrule(
name = "a",
outs = ["a.txt"],
cmd = "touch $(OUTS)",
)
EOF
stop_worker
start_worker --expected_authorization_token=secret_token
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
//a:a >& $TEST_log && fail "Build without credentials should have failed"
expect_log "Failed to query remote execution capabilities"
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--experimental_credential_helper="${TEST_TMPDIR}/credhelper" \
//a:a >& $TEST_log || fail "Build with credentials should have succeeded"
}
function test_remote_grpc_cache_with_protocol() {
# Test that if 'grpc' is provided as a scheme for --remote_cache flag, remote cache works.
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
)
EOF
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//a:foo \
|| fail "Failed to build //a:foo with remote cache"
}
# TODO(b/211478955): Deflake and re-enable.
function DISABLED_test_remote_grpc_via_unix_socket_proxy() {
case "$PLATFORM" in
darwin|freebsd|linux|openbsd)
;;
*)
return 0
;;
esac
# Test that remote execution can be routed via a UNIX domain socket if
# supported by the platform.
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
)
EOF
# Note: not using $TEST_TMPDIR because many OSes, notably macOS, have
# small maximum length limits for UNIX domain sockets.
socket_dir=$(mktemp -d -t "remote_executor.XXXXXXXX")
PROXY="$(rlocation io_bazel/src/test/shell/bazel/remote/uds_proxy.py)"
python "${PROXY}" "${socket_dir}/executor-socket" "localhost:${worker_port}" &
proxy_pid=$!
bazel build \
--remote_executor=grpc://noexist.invalid \
--remote_proxy="unix:${socket_dir}/executor-socket" \
//a:foo \
|| fail "Failed to build //a:foo with remote cache"
kill ${proxy_pid}
rm "${socket_dir}/executor-socket"
rmdir "${socket_dir}"
}
# TODO(b/211478955): Deflake and re-enable.
function DISABLED_test_remote_grpc_via_unix_socket_direct() {
case "$PLATFORM" in
darwin|freebsd|linux|openbsd)
;;
*)
return 0
;;
esac
# Test that remote execution can be routed via a UNIX domain socket if
# supported by the platform.
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
)
EOF
# Note: not using $TEST_TMPDIR because many OSes, notably macOS, have
# small maximum length limits for UNIX domain sockets.
socket_dir=$(mktemp -d -t "remote_executor.XXXXXXXX")
PROXY="$(rlocation io_bazel/src/test/shell/bazel/remote/uds_proxy.py)"
python "${PROXY}" "${socket_dir}/executor-socket" "localhost:${worker_port}" &
proxy_pid=$!
bazel build \
--remote_executor="unix:${socket_dir}/executor-socket" \
//a:foo \
|| fail "Failed to build //a:foo with remote cache"
kill ${proxy_pid}
rm "${socket_dir}/executor-socket"
rmdir "${socket_dir}"
}
function test_cc_binary() {
if [[ "$PLATFORM" == "darwin" ]]; then
# TODO(b/37355380): This test is disabled due to RemoteWorker not supporting
# setting SDKROOT and DEVELOPER_DIR appropriately, as is required of
# action executors in order to select the appropriate Xcode toolchain.
return 0
fi
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
cc_binary(
name = 'test',
srcs = [ 'test.cc' ],
)
EOF
cat > a/test.cc <<EOF
#include <iostream>
int main() { std::cout << "Hello world!" << std::endl; return 0; }
EOF
bazel build //a:test >& $TEST_log \
|| fail "Failed to build //a:test without remote execution"
cp -f bazel-bin/a/test ${TEST_TMPDIR}/test_expected
bazel clean >& $TEST_log
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
//a:test >& $TEST_log \
|| fail "Failed to build //a:test with remote execution"
expect_log "6 processes: 4 internal, 2 remote"
diff bazel-bin/a/test ${TEST_TMPDIR}/test_expected \
|| fail "Remote execution generated different result"
}
function test_cc_test() {
if [[ "$PLATFORM" == "darwin" ]]; then
# TODO(b/37355380): This test is disabled due to RemoteWorker not supporting
# setting SDKROOT and DEVELOPER_DIR appropriately, as is required of
# action executors in order to select the appropriate Xcode toolchain.
return 0
fi
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
cc_test(
name = 'test',
srcs = [ 'test.cc' ],
)
EOF
cat > a/test.cc <<EOF
#include <iostream>
int main() { std::cout << "Hello test!" << std::endl; return 0; }
EOF
bazel test \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--test_output=errors \
--noexperimental_split_xml_generation \
//a:test >& $TEST_log \
|| fail "Failed to run //a:test with remote execution"
}
function test_cc_test_split_xml() {
if [[ "$PLATFORM" == "darwin" ]]; then
# TODO(b/37355380): This test is disabled due to RemoteWorker not supporting
# setting SDKROOT and DEVELOPER_DIR appropriately, as is required of
# action executors in order to select the appropriate Xcode toolchain.
return 0
fi
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
cc_test(
name = 'test',
srcs = [ 'test.cc' ],
)
EOF
cat > a/test.cc <<EOF
#include <iostream>
int main() { std::cout << "Hello test!" << std::endl; return 0; }
EOF
bazel test \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--test_output=errors \
--experimental_split_xml_generation \
//a:test >& $TEST_log \
|| fail "Failed to run //a:test with remote execution"
}
function test_cc_binary_grpc_cache() {
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
cc_binary(
name = 'test',
srcs = [ 'test.cc' ],
)
EOF
cat > a/test.cc <<EOF
#include <iostream>
int main() { std::cout << "Hello world!" << std::endl; return 0; }
EOF
bazel build //a:test >& $TEST_log \
|| fail "Failed to build //a:test without remote cache"
cp -f bazel-bin/a/test ${TEST_TMPDIR}/test_expected
bazel clean >& $TEST_log
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//a:test >& $TEST_log \
|| fail "Failed to build //a:test with remote gRPC cache service"
diff bazel-bin/a/test ${TEST_TMPDIR}/test_expected \
|| fail "Remote cache generated different result"
}
function test_cc_binary_grpc_cache_statsline() {
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
cc_binary(
name = 'test',
srcs = [ 'test.cc' ],
)
EOF
cat > a/test.cc <<EOF
#include <iostream>
int main() { std::cout << "Hello world!" << std::endl; return 0; }
EOF
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//a:test >& $TEST_log \
|| fail "Failed to build //a:test with remote gRPC cache service"
bazel clean >& $TEST_log
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//a:test 2>&1 | tee $TEST_log | grep "remote cache hit" \
|| fail "Output does not contain remote cache hits"
}
function test_failing_cc_test() {
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
cc_test(
name = 'test',
srcs = [ 'test.cc' ],
)
EOF
cat > a/test.cc <<EOF
#include <iostream>
int main() { std::cout << "Fail me!" << std::endl; return 1; }
EOF
bazel test \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--test_output=errors \
//a:test >& $TEST_log \
&& fail "Expected test failure" || true
# TODO(ulfjack): Check that the test failure gets reported correctly.
}
function test_local_fallback_works_with_local_strategy() {
mkdir -p gen1
cat > gen1/BUILD <<'EOF'
genrule(
name = "gen1",
srcs = [],
outs = ["out1"],
cmd = "touch \"$@\"",
tags = ["no-remote"],
)
EOF
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--remote_local_fallback_strategy=local \
--build_event_text_file=gen1.log \
//gen1 >& $TEST_log \
&& fail "Expected failure" || true
}
function test_local_fallback_with_local_strategy_lists() {
mkdir -p gen1
cat > gen1/BUILD <<'EOF'
genrule(
name = "gen1",
srcs = [],
outs = ["out1"],
cmd = "touch \"$@\"",
tags = ["no-remote"],
)
EOF
bazel build \
--spawn_strategy=remote,local \
--remote_executor=grpc://localhost:${worker_port} \
--build_event_text_file=gen1.log \
//gen1 >& $TEST_log \
|| fail "Expected success"
mv gen1.log $TEST_log
expect_log "2 processes: 1 internal, 1 local"
}
function test_local_fallback_with_sandbox_strategy_lists() {
mkdir -p gen1
cat > gen1/BUILD <<'EOF'
genrule(
name = "gen1",
srcs = [],
outs = ["out1"],
cmd = "touch \"$@\"",
tags = ["no-remote"],
)
EOF
bazel build \
--spawn_strategy=remote,sandboxed,local \
--remote_executor=grpc://localhost:${worker_port} \
--build_event_text_file=gen1.log \
//gen1 >& $TEST_log \
|| fail "Expected success"
mv gen1.log $TEST_log
expect_log "2 processes: 1 internal, 1 .*-sandbox"
}
function test_local_fallback_to_sandbox_by_default() {
mkdir -p gen1
cat > gen1/BUILD <<'EOF'
genrule(
name = "gen1",
srcs = [],
outs = ["out1"],
cmd = "touch \"$@\"",
tags = ["no-remote"],
)
EOF
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
--build_event_text_file=gen1.log \
//gen1 >& $TEST_log \
|| fail "Expected success"
mv gen1.log $TEST_log
expect_log "2 processes: 1 internal, 1 .*-sandbox"
}
function test_local_fallback_works_with_sandboxed_strategy() {
mkdir -p gen2
cat > gen2/BUILD <<'EOF'
genrule(
name = "gen2",
srcs = [],
outs = ["out2"],
cmd = "touch \"$@\"",
tags = ["no-remote"],
)
EOF
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--remote_local_fallback_strategy=sandboxed \
--build_event_text_file=gen2.log \
//gen2 >& $TEST_log \
&& fail "Expected failure" || true
}
function test_local_fallback_if_no_remote_executor() {
# Test that when manually set --spawn_strategy that includes remote, but remote_executor isn't set, we ignore
# the remote strategy rather than reporting an error. See https://github.com/bazelbuild/bazel/issues/13340.
mkdir -p gen1
cat > gen1/BUILD <<'EOF'
genrule(
name = "gen1",
srcs = [],
outs = ["out1"],
cmd = "touch \"$@\"",
)
EOF
bazel build \
--spawn_strategy=remote,local \
--build_event_text_file=gen1.log \
//gen1 >& $TEST_log \
|| fail "Expected success"
mv gen1.log $TEST_log
expect_log "2 processes: 1 internal, 1 local"
}
function test_local_fallback_if_remote_executor_unavailable() {
# Test that when --remote_local_fallback is set and remote_executor is unavailable when build starts, we fallback to
# local strategy. See https://github.com/bazelbuild/bazel/issues/13487.
mkdir -p gen1
cat > gen1/BUILD <<'EOF'
genrule(
name = "gen1",
srcs = [],
outs = ["out1"],
cmd = "touch \"$@\"",
)
EOF
bazel build \
--spawn_strategy=remote,local \
--remote_executor=grpc://noexist.invalid \
--remote_local_fallback \
--build_event_text_file=gen1.log \
//gen1 >& $TEST_log \
|| fail "Expected success"
mv gen1.log $TEST_log
expect_log "2 processes: 1 internal, 1 local"
}
function is_file_uploaded() {
h=$(shasum -a256 < $1)
if [ -e "$cas_path/${h:0:64}" ]; then return 0; else return 1; fi
}
function test_failed_test_outputs_not_uploaded() {
# Test that outputs of a failed test/action are not uploaded to the remote
# cache. This is a regression test for https://github.com/bazelbuild/bazel/issues/7232
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
cc_test(
name = 'test',
srcs = [ 'test.cc' ],
)
EOF
cat > a/test.cc <<EOF
#include <iostream>
int main() { std::cout << "Fail me!" << std::endl; return 1; }
EOF
bazel test \
--remote_cache=grpc://localhost:${worker_port} \
--test_output=errors \
//a:test >& $TEST_log \
&& fail "Expected test failure" || true
($(is_file_uploaded bazel-testlogs/a/test/test.log) \
&& fail "Expected test log to not be uploaded to remote execution") || true
($(is_file_uploaded bazel-testlogs/a/test/test.xml) \
&& fail "Expected test xml to not be uploaded to remote execution") || true
}
# Tests that the remote worker can return a 200MB blob that requires chunking.
# Blob has to be that large in order to exceed the grpc default max message size.
function test_genrule_large_output_chunking() {
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
genrule(
name = "large_output",
srcs = ["small_blob.txt"],
outs = ["large_blob.txt"],
cmd = "cp \$(location small_blob.txt) tmp.txt; " +
"(for i in {1..22} ; do cat tmp.txt >> \$@; cp \$@ tmp.txt; done)",
)
EOF
cat > a/small_blob.txt <<EOF
0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
EOF
bazel build //a:large_output >& $TEST_log \
|| fail "Failed to build //a:large_output without remote execution"
cp -f bazel-genfiles/a/large_blob.txt ${TEST_TMPDIR}/large_blob_expected.txt
bazel clean >& $TEST_log
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
//a:large_output >& $TEST_log \
|| fail "Failed to build //a:large_output with remote execution"
diff bazel-genfiles/a/large_blob.txt ${TEST_TMPDIR}/large_blob_expected.txt \
|| fail "Remote execution generated different result"
}
function test_py_test() {
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
py_test(
name = 'test',
srcs = [ 'test.py' ],
)
EOF
cat > a/test.py <<'EOF'
import sys
if __name__ == "__main__":
sys.exit(0)
EOF
bazel test \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--test_output=errors \
//a:test >& $TEST_log \
|| fail "Failed to run //a:test with remote execution"
}
function test_py_test_with_xml_output() {
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
py_test(
name = 'test',
srcs = [ 'test.py' ],
)
EOF
cat > a/test.py <<'EOF'
import sys
import os
if __name__ == "__main__":
f = open(os.environ['XML_OUTPUT_FILE'], "w")
f.write('''
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite name="test" tests="1" failures="1" errors="1">
<testcase name="first" status="run">
<failure>That did not work!</failure>
</testcase>
</testsuite>
</testsuites>
''')
sys.exit(0)
EOF
bazel test \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--test_output=errors \
//a:test >& $TEST_log \
|| fail "Failed to run //a:test with remote execution"
xml=bazel-testlogs/a/test/test.xml
[ -e $xml ] || fail "Expected to find XML output"
cat $xml > $TEST_log
expect_log 'That did not work!'
}
function test_failing_py_test_with_xml_output() {
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
py_test(
name = 'test',
srcs = [ 'test.py' ],
)
EOF
cat > a/test.py <<'EOF'
import sys
import os
if __name__ == "__main__":
f = open(os.environ['XML_OUTPUT_FILE'], "w")
f.write('''
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite name="test" tests="1" failures="1" errors="1">
<testcase name="first" status="run">
<failure>That did not work!</failure>
</testcase>
</testsuite>
</testsuites>
''')
sys.exit(1)
EOF
bazel test \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--test_output=errors \
//a:test >& $TEST_log \
&& fail "Expected test failure" || true
xml=bazel-testlogs/a/test/test.xml
[ -e $xml ] || fail "Expected to find XML output"
cat $xml > $TEST_log
expect_log 'That did not work!'
}
function test_noinput_action() {
mkdir -p a
cat > a/rule.bzl <<'EOF'
def _impl(ctx):
output = ctx.outputs.out
ctx.actions.run_shell(
outputs=[output],
command="echo 'Hello World' > %s" % (output.path))
empty = rule(
implementation=_impl,
outputs={"out": "%{name}.txt"},
)
EOF
cat > a/BUILD <<'EOF'
load("//a:rule.bzl", "empty")
package(default_visibility = ["//visibility:public"])
empty(name = 'test')
EOF
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--test_output=errors \
//a:test >& $TEST_log \
|| fail "Failed to run //a:test with remote execution"
}
function test_timeout() {
mkdir -p a
cat > a/BUILD <<'EOF'
sh_test(
name = "sleep",
timeout = "short",
srcs = ["sleep.sh"],
)
EOF
cat > a/sleep.sh <<'EOF'
#!/bin/bash
for i in {1..3}
do
echo "Sleeping $i..."
sleep 1
done
EOF
chmod +x a/sleep.sh
bazel test \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--test_output=errors \
--test_timeout=1,1,1,1 \
//a:sleep >& $TEST_log \
&& fail "Test failure (timeout) expected" || true
expect_log "TIMEOUT"
expect_log "Sleeping 1..."
# The current implementation of the remote worker does not terminate actions
# when they time out, therefore we cannot verify that:
# expect_not_log "Sleeping 3..."
}
function test_passed_env_user() {
mkdir -p a
cat > a/BUILD <<'EOF'
sh_test(
name = "user_test",
timeout = "short",
srcs = ["user_test.sh"],
)
EOF
cat > a/user_test.sh <<'EOF'
#!/bin/sh
echo "user=$USER"
EOF
chmod +x a/user_test.sh
bazel test \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--test_output=all \
--test_env=USER=boo \
//a:user_test >& $TEST_log \
|| fail "Failed to run //a:user_test with remote execution"
expect_log "user=boo"
# Rely on the test-setup script to set USER value to whoami.
export USER=
bazel test \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--test_output=all \
//a:user_test >& $TEST_log \
|| fail "Failed to run //a:user_test with remote execution"
expect_log "user=$(whoami)"
}
function test_exitcode() {
mkdir -p a
cat > a/BUILD <<'EOF'
genrule(
name = "foo",
srcs = [],
outs = ["foo.txt"],
cmd = "echo \"hello world\" > \"$@\"",
)
EOF
(set +e
bazel build \
--genrule_strategy=remote \
--remote_executor=bazel-test-does-not-exist \
//a:foo >& $TEST_log
[ $? -eq 34 ]) || fail "Test failed due to wrong exit code"
}
# Bazel should display non-test errors to the user, instead of hiding them behind test failures.
# For example, if the network connection to the remote executor fails it shouldn't be displayed as
# a test error.
function test_display_non_testerrors() {
mkdir -p a
cat > a/BUILD <<'EOF'
sh_test(
name = "test",
timeout = "short",
srcs = ["test.sh"],
)
EOF
cat > a/test.sh <<'EOF'
#!/bin/sh
#This will never run, because the remote side is not reachable.
EOF
chmod +x a/test.sh
bazel test \
--spawn_strategy=remote \
--remote_executor=grpc://bazel.does.not.exist:1234 \
--remote_retries=0 \
--test_output=all \
--test_env=USER=boo \
//a:test >& $TEST_log \
&& fail "Test failure expected" || true
expect_not_log "test.log"
expect_log "Failed to query remote execution capabilities"
}
function set_symlinks_in_directory_testfixtures() {
cat > BUILD <<'EOF'
genrule(
name = 'make-links',
outs = ['dir', 'r', 'a', 'rd', 'ad'],
cmd = ('mkdir $(location dir) && ' +
'cd $(location dir) && ' +
'echo hello > foo && ' + # Regular file.
'ln -s foo r && ' + # Relative symlink, will be passed as symlink.
'ln -s $$PWD/foo a && ' + # Absolute symlink, will be copied.
'mkdir bar && ' + # Regular directory.
'echo bla > bar/baz && ' +
'ln -s bar rd && ' + # Relative symlink, will be passed as symlink.
'ln -s $$PWD/bar ad && ' + # Absolute symlink, will be copied.
'cd .. && ' +
'ln -s dir/foo r && ' + # Relative symlink, will be passed as symlink.
'ln -s $$PWD/dir/foo a && ' + # Absolute symlink, will be copied.
'ln -s dir rd && ' + # Relative symlink, will be passed as symlink.
'ln -s $$PWD/dir ad' # Absolute symlink, will be copied.
),
)
EOF
cat > "${TEST_TMPDIR}/expected_links" <<'EOF'
./ad/r
./ad/rd
./dir/r
./dir/rd
./r
./rd
EOF
}
function test_symlinks_in_directory() {
set_symlinks_in_directory_testfixtures
bazel build \
--incompatible_remote_symlinks \
--remote_executor=grpc://localhost:${worker_port} \
--spawn_strategy=remote \
//:make-links &> $TEST_log \
|| fail "Failed to build //:make-links with remote execution"
expect_log "1 remote"
find -L bazel-genfiles -type f -exec cat {} \; | sort | uniq -c &> $TEST_log
expect_log "9 bla"
expect_log "11 hello"
CUR=$PWD && cd bazel-genfiles && \
find . -type l | sort > "${TEST_TMPDIR}/links" && cd $CUR
diff "${TEST_TMPDIR}/links" "${TEST_TMPDIR}/expected_links" \
|| fail "Remote execution created different symbolic links"
}
function test_symlinks_in_directory_cache_only() {
# This test is the same as test_symlinks_in_directory, except it works
# locally and uses the remote cache to query results.
set_symlinks_in_directory_testfixtures
bazel build \
--incompatible_remote_symlinks \
--remote_cache=grpc://localhost:${worker_port} \
--spawn_strategy=local \
//:make-links &> $TEST_log \
|| fail "Failed to build //:make-links with remote cache service"
expect_log "1 local"
bazel clean # Get rid of local results, rely on remote cache.
bazel build \
--incompatible_remote_symlinks \
--remote_cache=grpc://localhost:${worker_port} \
--spawn_strategy=local \
//:make-links &> $TEST_log \
|| fail "Failed to build //:make-links with remote cache service"
expect_log "1 remote cache hit"
# Check that the results downloaded from remote cache are the same as local.
find -L bazel-genfiles -type f -exec cat {} \; | sort | uniq -c &> $TEST_log
expect_log "9 bla"
expect_log "11 hello"
CUR=$PWD && cd bazel-genfiles && \
find . -type l | sort > "${TEST_TMPDIR}/links" && cd $CUR
diff "${TEST_TMPDIR}/links" "${TEST_TMPDIR}/expected_links" \
|| fail "Cached result created different symbolic links"
}
function test_treeartifact_in_runfiles() {
mkdir -p a
cat > a/BUILD <<'EOF'
load(":output_directory.bzl", "gen_output_dir", "gen_output_dir_test")
gen_output_dir(
name = "starlark_output_dir",
outdir = "dir",
)
gen_output_dir_test(
name = "starlark_output_dir_test",
dir = ":starlark_output_dir",
)
EOF
cat > a/output_directory.bzl <<'EOF'
def _gen_output_dir_impl(ctx):
output_dir = ctx.actions.declare_directory(ctx.attr.outdir)
ctx.actions.run_shell(
outputs = [output_dir],
inputs = [],
command = """
mkdir -p $1/sub; \
echo "foo" > $1/foo; \
echo "bar" > $1/sub/bar
""",
arguments = [output_dir.path],
)
return [
DefaultInfo(files=depset(direct=[output_dir]),
runfiles = ctx.runfiles(files = [output_dir]))
]
gen_output_dir = rule(
implementation = _gen_output_dir_impl,
attrs = {
"outdir": attr.string(mandatory = True),
},
)
def _gen_output_dir_test_impl(ctx):
test = ctx.actions.declare_file(ctx.label.name)
ctx.actions.write(test, "echo hello world")
myrunfiles = ctx.runfiles(files=ctx.attr.dir.default_runfiles.files.to_list())
return [
DefaultInfo(
executable = test,
runfiles = myrunfiles,
),
]
gen_output_dir_test = rule(
implementation = _gen_output_dir_test_impl,
test = True,
attrs = {
"dir": attr.label(mandatory = True),
},
)
EOF
# Also test this directory inputs with sandboxing. Ideally we would add such
# a test into the sandboxing module.
bazel test \
--spawn_strategy=sandboxed \
//a:starlark_output_dir_test \
|| fail "Failed to run //a:starlark_output_dir_test with sandboxing"
bazel test \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
//a:starlark_output_dir_test \
|| fail "Failed to run //a:starlark_output_dir_test with remote execution"
}
function test_remote_exec_properties() {
# Test that setting remote exec properties works.
mkdir -p a
cat > a/BUILD <<'EOF'
genrule(
name = "foo",
srcs = [],
outs = ["foo.txt"],
cmd = "echo \"foo\" > \"$@\"",
)
EOF
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
--remote_default_exec_properties=OSFamily=linux \
//a:foo || fail "Failed to build //a:foo"
bazel clean
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
--remote_default_exec_properties=OSFamily=windows \
//a:foo >& $TEST_log || fail "Failed to build //a:foo"
expect_not_log "remote cache hit"
}
function test_tag_no_cache() {
mkdir -p a
cat > a/BUILD <<'EOF'
genrule(
name = "foo",
srcs = [],
outs = ["foo.txt"],
cmd = "echo \"foo\" > \"$@\"",
tags = ["no-cache"]
)
EOF
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
//a:foo >& $TEST_log || "Failed to build //a:foo"
expect_log "1 remote"
bazel clean
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
//a:foo >& $TEST_log || "Failed to build //a:foo"
expect_log "1 remote"
expect_not_log "remote cache hit"
}
function test_tag_no_cache_for_disk_cache() {
mkdir -p a
cat > a/BUILD <<'EOF'
genrule(
name = "foo",
srcs = [],
outs = ["foo.txt"],
cmd = "echo \"foo\" > \"$@\"",
tags = ["no-cache"]
)
EOF
CACHEDIR=$(mktemp -d)
bazel build \
--disk_cache=$CACHEDIR \
//a:foo >& $TEST_log || "Failed to build //a:foo"
expect_log "1 .*-sandbox"
bazel clean
bazel build \
--disk_cache=$CACHEDIR \
//a:foo >& $TEST_log || "Failed to build //a:foo"
expect_log "1 .*-sandbox"
expect_not_log "remote cache hit"
}
function test_tag_no_remote_cache() {
mkdir -p a
cat > a/BUILD <<'EOF'
genrule(
name = "foo",
srcs = [],
outs = ["foo.txt"],
cmd = "echo \"foo\" > \"$@\"",
tags = ["no-remote-cache"]
)
EOF
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
//a:foo >& $TEST_log || "Failed to build //a:foo"
expect_log "1 remote"
bazel clean
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
//a:foo >& $TEST_log || "Failed to build //a:foo"
expect_log "1 remote"
expect_not_log "remote cache hit"
}
function test_tag_no_remote_cache_upload() {
mkdir -p a
cat > a/BUILD <<'EOF'
genrule(
name = "foo",
srcs = [],
outs = ["foo.txt"],
cmd = "echo \"foo\" > \"$@\"",
)
EOF
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--modify_execution_info=.*=+no-remote-cache-upload \
//a:foo >& $TEST_log || "Failed to build //a:foo"
remote_ac_files="$(count_remote_ac_files)"
[[ "$remote_ac_files" == 0 ]] || fail "Expected 0 remote action cache entries, not $remote_ac_files"
# populate the cache
bazel clean
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//a:foo >& $TEST_log || "Failed to build //a:foo"
bazel clean
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--modify_execution_info=.*=+no-remote-cache-upload \
//a:foo >& $TEST_log || "Failed to build //a:foo"
expect_log "remote cache hit"
}
function test_tag_no_remote_cache_for_disk_cache() {
mkdir -p a
cat > a/BUILD <<'EOF'
genrule(
name = "foo",
srcs = [],
outs = ["foo.txt"],
cmd = "echo \"foo\" > \"$@\"",
tags = ["no-remote-cache"]
)
EOF
CACHEDIR=$(mktemp -d)
bazel build \
--disk_cache=$CACHEDIR \
//a:foo >& $TEST_log || "Failed to build //a:foo"
expect_log "1 .*-sandbox"
bazel clean
bazel build \
--disk_cache=$CACHEDIR \
//a:foo >& $TEST_log || "Failed to build //a:foo"
expect_log "1 disk cache hit"
}
function test_tag_no_remote_exec() {
mkdir -p a
cat > a/BUILD <<'EOF'
genrule(
name = "foo",
srcs = [],
outs = ["foo.txt"],
cmd = "echo \"foo\" > \"$@\"",
tags = ["no-remote-exec"]
)
EOF
bazel build \
--spawn_strategy=remote,local \
--remote_executor=grpc://localhost:${worker_port} \
//a:foo >& $TEST_log || "Failed to build //a:foo"
expect_log "1 local"
expect_not_log "1 remote"
bazel clean
bazel build \
--spawn_strategy=remote,local \
--remote_executor=grpc://localhost:${worker_port} \
//a:foo >& $TEST_log || "Failed to build //a:foo"
expect_log "1 remote cache hit"
expect_not_log "1 local"
}
function test_nobuild_runfile_links() {
mkdir data && echo "hello" > data/hello && echo "world" > data/world
cat > WORKSPACE <<EOF
workspace(name = "foo")
EOF
cat > test.sh <<'EOF'
#!/bin/bash
set -e
[[ -f ${RUNFILES_DIR}/foo/data/hello ]]
[[ -f ${RUNFILES_DIR}/foo/data/world ]]
exit 0
EOF
chmod 755 test.sh
cat > BUILD <<'EOF'
filegroup(
name = "runfiles",
srcs = ["data/hello", "data/world"],
)
sh_test(
name = "test",
srcs = ["test.sh"],
data = [":runfiles"],
)
EOF
bazel test \
--nobuild_runfile_links \
--remote_executor=grpc://localhost:${worker_port} \
//:test || fail "Testing //:test failed"
[[ ! -f bazel-bin/test.runfiles/foo/data/hello ]] || fail "expected no runfile data/hello"
[[ ! -f bazel-bin/test.runfiles/foo/data/world ]] || fail "expected no runfile data/world"
[[ ! -f bazel-bin/test.runfiles/MANIFEST ]] || fail "expected output manifest to exist"
}
function test_platform_default_properties_invalidation() {
# Test that when changing values of --remote_default_platform_properties all actions are
# invalidated.
mkdir -p test
cat > test/BUILD << 'EOF'
genrule(
name = "test",
srcs = [],
outs = ["output.txt"],
cmd = "echo \"foo\" > \"$@\""
)
EOF
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
--remote_default_exec_properties="build=1234" \
//test:test >& $TEST_log || fail "Failed to build //a:remote"
expect_log "2 processes: 1 internal, 1 remote"
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
--remote_default_exec_properties="build=88888" \
//test:test >& $TEST_log || fail "Failed to build //a:remote"
# Changing --remote_default_platform_properties value should invalidate SkyFrames in-memory
# caching and make it re-run the action.
expect_log "2 processes: 1 internal, 1 remote"
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
--remote_default_exec_properties="build=88888" \
//test:test >& $TEST_log || fail "Failed to build //a:remote"
# The same value of --remote_default_platform_properties should NOT invalidate SkyFrames in-memory cache
# and make the action should not be re-run.
expect_log "1 process: 1 internal"
bazel shutdown
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
--remote_default_exec_properties="build=88888" \
//test:test >& $TEST_log || fail "Failed to build //a:remote"
# The same value of --remote_default_platform_properties should NOT invalidate SkyFrames od-disk cache
# and the action should not be re-run.
expect_log "1 process: 1 internal"
bazel build\
--remote_executor=grpc://localhost:${worker_port} \
--remote_default_exec_properties="build=88888" \
--remote_default_platform_properties='properties:{name:"build" value:"1234"}' \
//test:test >& $TEST_log && fail "Should fail" || true
# Build should fail with a proper error message if both
# --remote_default_platform_properties and --remote_default_exec_properties
# are provided via command line
expect_log "Setting both --remote_default_platform_properties and --remote_default_exec_properties is not allowed"
}
function test_combined_disk_remote_exec_with_flag_combinations() {
rm -f ${TEST_TMPDIR}/test_expected
declare -a testcases=(
# ensure CAS entries get uploaded even when action entries don't.
"--noremote_upload_local_results"
"--remote_upload_local_results"
# we should see no cache hits [incompatible_remote_results_ignore_disk=false is default]
"--noremote_accept_cached"
# Should be some disk cache hits, just not remote.
"--noremote_accept_cached --incompatible_remote_results_ignore_disk"
)
for flags in "${testcases[@]}"; do
genrule_combined_disk_remote_exec "$flags"
# clean up and start a new worker for the next run
tear_down
set_up
done
}
function genrule_combined_disk_remote_exec() {
# Test for the combined disk and grpc cache with remote_exec
# These flags get reset before the bazel runs when we clear caches.
local cache="${TEST_TMPDIR}/disk_cache"
local disk_flags="--disk_cache=$cache"
local grpc_flags="--remote_cache=grpc://localhost:${worker_port}"
local remote_exec_flags="--remote_executor=grpc://localhost:${worker_port}"
# These flags are the same for all bazel runs.
local testcase_flags="$@"
local spawn_flags=("--spawn_strategy=remote" "--genrule_strategy=remote")
# if exist in disk cache or remote cache, don't run remote exec, don't update caches.
# [CASE]disk_cache, remote_cache: remote_exec, disk_cache, remote_cache
# 1) notexist notexist run OK update, update
# 2) notexist exist no run update, no update
# 3) exist notexist no run no update, no update
# 4) exist exist no run no update, no update
# 5) another rule that depends on 4, but run before 5
# Our setup ensures the first 2 columns, our validation checks the last 3.
# NOTE that remote_exec will UPDATE the disk cache.
#
# We measure if it was run remotely via the "1 remote." in the output and caches
# from the cache hit on the same line.
# https://cs.opensource.google/bazel/bazel/+/master:third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto;l=447;drc=29ac010f3754c308de2ff13d3480b870dc7cb7f6
#
# tags: [nocache, noremoteexec]
mkdir -p a
cat > a/BUILD <<'EOF'
package(default_visibility = ["//visibility:public"])
genrule(
name = 'test',
cmd = 'echo "Hello world" > $@',
outs = ['test.txt'],
)
genrule(
name = 'test2',
srcs = [':test'],
cmd = 'cat $(SRCS) > $@',
outs = ['test2.txt'],
)
EOF
rm -rf $cache
mkdir $cache
echo "INFO: RUNNING testcase($testcase_flags)"
# Case 1)
# disk_cache, remote_cache: remote_exec, disk_cache, remote_cache
# notexist notexist run OK update, update
#
# Do a build to populate the disk and remote cache.
# Then clean and do another build to validate nothing updates.
bazel build $spawn_flags $testcase_flags $remote_exec_flags $grpc_flags $disk_flags //a:test &> $TEST_log \
|| fail "CASE 1 Failed to build"
echo "Hello world" > ${TEST_TMPDIR}/test_expected
expect_log "2 processes: 1 internal, 1 remote." "CASE 1: unexpected action line [[$(grep processes $TEST_log)]]"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "Disk cache generated different result [$(cat bazel-genfiles/a/test.txt)] [$(cat $TEST_TMPDIR/test_expected)]"
disk_action_cache_files="$(count_disk_ac_files "$cache")"
remote_action_cache_files="$(count_remote_ac_files)"
[[ "$disk_action_cache_files" == 1 ]] || fail "CASE 1: Expected 1 disk action cache entries, not $disk_action_cache_files"
# Even though bazel isn't writing the remote action cache, we expect the worker to write one or the
# the rest of our tests will fail.
[[ "$remote_action_cache_files" == 1 ]] || fail "CASE 1: Expected 1 remote action cache entries, not $remote_action_cache_files"
rm -rf $cache
mkdir $cache
# Case 2)
# disk_cache, remote_cache: remote_exec, disk_cache, remote_cache
# notexist exist no run update, no update
bazel clean
bazel build $spawn_flags $testcase_flags $remote_exec_flags $grpc_flags $disk_flags //a:test &> $TEST_log \
|| fail "CASE 2 Failed to build"
if [[ "$testcase_flags" == --noremote_accept_cached* ]]; then
expect_log "2 processes: 1 internal, 1 remote." "CASE 2a: unexpected action line [[$(grep processes $TEST_log)]]"
else
expect_log "2 processes: 1 remote cache hit, 1 internal." "CASE 2: unexpected action line [[$(grep processes $TEST_log)]]"
fi
# ensure disk and remote cache populated
disk_action_cache_files="$(count_disk_ac_files "$cache")"
remote_action_cache_files="$(count_remote_ac_files)"
[[ "$disk_action_cache_files" == 1 ]] || fail "CASE 2: Expected 1 disk action cache entries, not $disk_action_cache_files"
[[ "$remote_action_cache_files" == 1 ]] || fail "CASE 2: Expected 1 remote action cache entries, not $remote_action_cache_files"
# Case 3)
# disk_cache, remote_cache: remote_exec, disk_cache, remote_cache
# exist notexist no run no update, no update
# stop the worker to clear the remote cache and then restart it.
# This ensures that if we hit the disk cache and it returns valid values
# for FindMissingBLobs, the remote exec can still find it from the remote cache.
stop_worker
start_worker
# need to reset flags after restarting worker [on new port]
local grpc_flags="--remote_cache=grpc://localhost:${worker_port}"
local remote_exec_flags="--remote_executor=grpc://localhost:${worker_port}"
bazel clean
bazel build $spawn_flags $testcase_flags $remote_exec_flags $grpc_flags $disk_flags //a:test &> $TEST_log \
|| fail "CASE 3 failed to build"
if [[ "$testcase_flags" == --noremote_accept_cached ]]; then
expect_log "2 processes: 1 internal, 1 remote." "CASE 3: unexpected action line [[$(grep processes $TEST_log)]]"
else
expect_log "2 processes: 1 disk cache hit, 1 internal." "CASE 3: unexpected action line [[$(grep processes $TEST_log)]]"
fi
# Case 4)
# disk_cache, remote_cache: remote_exec, disk_cache, remote_cache
# exist exist no run no update, no update
# This one is not interesting after case 3.
bazel clean
bazel build $spawn_flags $testcase_flags $remote_exec_flags $grpc_flags $disk_flags //a:test &> $TEST_log \
|| fail "CASE 4 failed to build"
if [[ "$testcase_flags" == --noremote_accept_cached ]]; then
expect_log "2 processes: 1 internal, 1 remote." "CASE 4: unexpected action line [[$(grep processes $TEST_log)]]"
else
expect_log "2 processes: 1 disk cache hit, 1 internal." "CASE 4: unexpected action line [[$(grep processes $TEST_log)]]"
fi
# One last slightly more complicated case.
# Build a target that depended on the last target but we clean and clear the remote cache.
# We should get one cache hit from disk and and one remote exec.
stop_worker
start_worker
# reset port
local grpc_flags="--remote_cache=grpc://localhost:${worker_port}"
local remote_exec_flags="--remote_executor=grpc://localhost:${worker_port}"
bazel clean
bazel build $spawn_flags $testcase_flags --genrule_strategy=remote $remote_exec_flags $grpc_flags $disk_flags //a:test2 &> $TEST_log \
|| fail "CASE 5 failed to build //a:test2"
if [[ "$testcase_flags" == --noremote_accept_cached ]]; then
expect_log "3 processes: 1 internal, 2 remote." "CASE 5: unexpected action line [[$(grep processes $TEST_log)]]"
else
expect_log "3 processes: 1 disk cache hit, 1 internal, 1 remote." "CASE 5: unexpected action line [[$(grep processes $TEST_log)]]"
fi
}
function test_combined_disk_remote_exec_nocache_tag() {
rm -rf ${TEST_TMPDIR}/test_expected
local cache="${TEST_TMPDIR}/disk_cache"
local flags=("--disk_cache=$cache"
"--remote_cache=grpc://localhost:${worker_port}"
"--remote_executor=grpc://localhost:${worker_port}"
"--spawn_strategy=remote"
"--genrule_strategy=remote")
mkdir -p a
cat > a/BUILD <<'EOF'
package(default_visibility = ["//visibility:public"])
genrule(
name = 'nocache_test',
cmd = 'echo "Hello world" > $@',
outs = ['test.txt'],
tags = ['no-cache'],
)
EOF
rm -rf $cache
mkdir $cache
bazel build "${flags[@]}" //a:nocache_test &> $TEST_log \
|| fail "CASE 1 Failed to build"
echo "Hello world" > ${TEST_TMPDIR}/test_expected
expect_log "2 processes: 1 internal, 1 remote." "CASE 1: unexpected action line [[$(grep processes $TEST_log)] $flags]"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "different result 1 [$(cat bazel-bin/a/test.txt)] [$(cat $TEST_TMPDIR/test_expected)]"
# build it again, there should be no caching
bazel clean
bazel build "${flags[@]}" //a:nocache_test &> $TEST_log \
|| fail "CASE 2 Failed to build"
ls -l bazel-bin/a
expect_log "2 processes: 1 internal, 1 remote." "CASE 2: unexpected action line [[$(grep processes $TEST_log)]]"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "different result 2 [$(cat bazel-bin/a/test.txt)] [$(cat $TEST_TMPDIR/test_expected)]"
}
function test_genrule_combined_disk_grpc_cache() {
# Test for the combined disk and grpc cache.
# Built items should be pushed to both the disk and grpc cache.
# If --noremote_upload_local_results flag is set,
# built items should only be pushed to the disk cache.
# If --noremote_accept_cached flag is set,
# built items should only be checked from the disk cache.
# If an item is missing on disk cache, but present on grpc cache,
# then bazel should copy it from grpc cache to disk cache on fetch.
local cache="${TEST_TMPDIR}/cache"
local disk_flags="--disk_cache=$cache"
local grpc_flags="--remote_cache=grpc://localhost:${worker_port}"
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
genrule(
name = 'test',
cmd = 'echo "Hello world" > \$@',
outs = [ 'test.txt' ],
)
EOF
rm -rf $cache
mkdir $cache
# Build and push to disk cache but not grpc cache
bazel build $disk_flags $grpc_flags --incompatible_remote_results_ignore_disk=true --noremote_upload_local_results //a:test \
|| fail "Failed to build //a:test with combined disk grpc cache"
cp -f bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected
# Fetch from disk cache
bazel clean
bazel build $disk_flags //a:test --incompatible_remote_results_ignore_disk=true --noremote_upload_local_results &> $TEST_log \
|| fail "Failed to fetch //a:test from disk cache"
expect_log "1 disk cache hit" "Fetch from disk cache failed"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "Disk cache generated different result"
# No cache result from grpc cache, rebuild target
bazel clean
bazel build $grpc_flags //a:test --incompatible_remote_results_ignore_disk=true --noremote_upload_local_results &> $TEST_log \
|| fail "Failed to build //a:test"
expect_not_log "1 remote cache hit" "Should not get cache hit from grpc cache"
expect_log "1 .*-sandbox" "Rebuild target failed"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "Rebuilt target generated different result"
rm -rf $cache
mkdir $cache
# No cache result from grpc cache, rebuild target, and upload result to grpc cache
bazel clean
bazel build $grpc_flags //a:test --incompatible_remote_results_ignore_disk=true --noremote_accept_cached &> $TEST_log \
|| fail "Failed to build //a:test"
expect_not_log "1 remote cache hit" "Should not get cache hit from grpc cache"
expect_log "1 .*-sandbox" "Rebuild target failed"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "Rebuilt target generated different result"
# No cache result from grpc cache, rebuild target, and upload result to disk cache
bazel clean
bazel build $disk_flags $grpc_flags //a:test --incompatible_remote_results_ignore_disk=true --noremote_accept_cached &> $TEST_log \
|| fail "Failed to build //a:test"
expect_not_log "1 remote cache hit" "Should not get cache hit from grpc cache"
expect_log "1 .*-sandbox" "Rebuild target failed"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "Rebuilt target generated different result"
# Fetch from disk cache
bazel clean
bazel build $disk_flags $grpc_flags //a:test --incompatible_remote_results_ignore_disk=true --noremote_accept_cached &> $TEST_log \
|| fail "Failed to build //a:test"
expect_log "1 disk cache hit" "Fetch from disk cache failed"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "Disk cache generated different result"
rm -rf $cache
mkdir $cache
# Build and push to disk cache and grpc cache
bazel clean
bazel build $disk_flags $grpc_flags //a:test \
|| fail "Failed to build //a:test with combined disk grpc cache"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "Built target generated different result"
# Fetch from disk cache
bazel clean
bazel build $disk_flags //a:test &> $TEST_log \
|| fail "Failed to fetch //a:test from disk cache"
expect_log "1 disk cache hit" "Fetch from disk cache failed"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "Disk cache generated different result"
# Fetch from grpc cache
bazel clean
bazel build $grpc_flags //a:test &> $TEST_log \
|| fail "Failed to fetch //a:test from grpc cache"
expect_log "1 remote cache hit" "Fetch from grpc cache failed"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "HTTP cache generated different result"
rm -rf $cache
mkdir $cache
# Copy from grpc cache to disk cache
bazel clean
bazel build $disk_flags $grpc_flags //a:test &> $TEST_log \
|| fail "Failed to copy //a:test from grpc cache to disk cache"
expect_log "1 remote cache hit" "Copy from grpc cache to disk cache failed"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "HTTP cache generated different result"
# Fetch from disk cache
bazel clean
bazel build $disk_flags //a:test &> $TEST_log \
|| fail "Failed to fetch //a:test from disk cache"
expect_log "1 disk cache hit" "Fetch from disk cache after copy from grpc cache failed"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "Disk cache generated different result"
rm -rf $cache
}
function test_combined_cache_upload() {
mkdir -p a
echo 'bar' > a/bar
cat > a/BUILD <<'EOF'
genrule(
name = "foo",
srcs = [":bar"],
outs = ["foo.txt"],
cmd = """
echo $(location :bar) > "$@"
""",
)
EOF
CACHEDIR=$(mktemp -d)
bazel build \
--disk_cache=$CACHEDIR \
--remote_cache=grpc://localhost:${worker_port} \
//a:foo >& $TEST_log || "Failed to build //a:foo"
remote_ac_files="$(count_remote_ac_files)"
[[ "$remote_ac_files" == 1 ]] || fail "Expected 1 remote action cache entries, not $remote_ac_files"
remote_cas_files="$(count_remote_cas_files)"
[[ "$remote_cas_files" == 3 ]] || fail "Expected 3 remote cas entries, not $remote_cas_files"
}
function test_combined_cache_with_no_remote_cache_tag_remote_cache() {
# Test that actions with no-remote-cache tag can hit disk cache of a combined cache but
# remote cache is disabled.
combined_cache_with_no_remote_cache_tag "--remote_cache=grpc://localhost:${worker_port}"
}
function test_combined_cache_with_no_remote_cache_tag_remote_execution() {
# Test that actions with no-remote-cache tag can hit disk cache of a combined cache but
# remote cache is disabled.
combined_cache_with_no_remote_cache_tag "--remote_executor=grpc://localhost:${worker_port}"
}
function combined_cache_with_no_remote_cache_tag() {
local cache="${TEST_TMPDIR}/cache"
local disk_flags="--disk_cache=$cache"
local grpc_flags="$@"
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
genrule(
name = 'test',
cmd = 'echo "Hello world" > \$@',
outs = [ 'test.txt' ],
tags = ['no-remote-cache'],
)
EOF
rm -rf $cache
mkdir $cache
# Build and push to disk cache but not remote cache
bazel build $disk_flags $grpc_flags --incompatible_remote_results_ignore_disk=true //a:test \
|| fail "Failed to build //a:test with combined cache"
cp -f bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected
# Fetch from disk cache
bazel clean
bazel build $disk_flags //a:test --incompatible_remote_results_ignore_disk=true &> $TEST_log \
|| fail "Failed to fetch //a:test from disk cache"
expect_log "1 disk cache hit" "Fetch from disk cache failed"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "Disk cache generated different result"
# No cache result from grpc cache, rebuild target
bazel clean
bazel build $grpc_flags //a:test --incompatible_remote_results_ignore_disk=true &> $TEST_log \
|| fail "Failed to build //a:test"
expect_not_log "1 remote cache hit" "Should not get cache hit from grpc cache"
diff bazel-genfiles/a/test.txt ${TEST_TMPDIR}/test_expected \
|| fail "Rebuilt target generated different result"
}
function test_combined_cache_with_no_remote_tag() {
# Test that outputs of actions tagged with no-remote should not be uploaded
# to remote cache when remote execution is enabled. See
# https://github.com/bazelbuild/bazel/issues/14900.
mkdir -p a
cat > a/BUILD <<EOF
package(default_visibility = ["//visibility:public"])
genrule(
name = 'test',
cmd = 'echo "Hello world" > \$@',
outs = [ 'test.txt' ],
tags = ['no-remote'],
)
EOF
cache_dir=$(mktemp -d)
bazel build \
--disk_cache=${cache_dir} \
--remote_executor=grpc://localhost:${worker_port} \
--incompatible_remote_results_ignore_disk=true \
//a:test &> $TEST_log \
|| fail "Failed to build //a:test"
remote_ac_files="$(count_remote_ac_files)"
[[ "$remote_ac_files" == 0 ]] || fail "Expected 0 remote action cache entries, not $remote_ac_files"
remote_cas_files="$(count_remote_cas_files)"
[[ "$remote_cas_files" == 0 ]] || fail "Expected 0 remote cas entries, not $remote_cas_files"
}
function test_repo_remote_exec() {
# Test that repository_ctx.execute can execute a command remotely.
touch BUILD
cat > test.bzl <<'EOF'
def _impl(ctx):
res = ctx.execute(["/bin/bash", "-c", "echo -n $BAZEL_REMOTE_PLATFORM"])
if res.return_code != 0:
fail("Return code 0 expected, but was " + res.return_code)
entries = res.stdout.split(",")
if len(entries) != 2:
fail("Two platform kv pairs expected. Got:" + str(entries))
if entries[0] != "ISA=x86-64":
fail("'ISA' expected in remote platform'")
if entries[1] != "OSFamily=Linux":
fail("'OSFamily' expected in remote platform'")
ctx.file("BUILD")
foo_configure = repository_rule(
implementation = _impl,
remotable = True,
)
EOF
cat > WORKSPACE <<'EOF'
load("//:test.bzl", "foo_configure")
foo_configure(
name = "default_foo",
exec_properties = {
"OSFamily" : "Linux",
"ISA" : "x86-64",
}
)
EOF
bazel fetch \
--remote_executor=grpc://localhost:${worker_port} \
--experimental_repo_remote_exec \
@default_foo//:all
}
function test_repo_remote_exec_path_argument() {
# Test that repository_ctx.execute fails with a descriptive error message
# if a path argument is provided. The upload of files as part of command
# execution is not yet supported.
touch BUILD
echo "hello world" > input.txt
cat > test.bzl <<'EOF'
def _impl(ctx):
ctx.execute(["cat", ctx.path("input.txt")])
ctx.file("BUILD")
foo_configure = repository_rule(
implementation = _impl,
remotable = True,
)
EOF
cat > WORKSPACE <<'EOF'
load("//:test.bzl", "foo_configure")
foo_configure(
name = "default_foo",
)
EOF
bazel fetch \
--remote_executor=grpc://localhost:${worker_port} \
--experimental_repo_remote_exec \
@default_foo//:all >& $TEST_log && fail "Should fail" || true
expect_log "Argument 1 of execute is neither a label nor a string"
}
function test_repo_remote_exec_timeout() {
# Test that a remote job is killed if it exceeds the timeout.
touch BUILD
cat > test.bzl <<'EOF'
def _impl(ctx):
ctx.execute(["/bin/bash","-c",
"for i in {1..3}; do echo \"Sleeping $i...\" && sleep 1; done"], timeout=1)
ctx.file("BUILD")
foo_configure = repository_rule(
implementation = _impl,
remotable = True,
)
EOF
cat > WORKSPACE <<'EOF'
load("//:test.bzl", "foo_configure")
foo_configure(
name = "default_foo",
)
EOF
bazel fetch \
--remote_executor=grpc://localhost:${worker_port} \
--experimental_repo_remote_exec \
@default_foo//:all >& $TEST_log && fail "Should fail" || true
expect_log "exceeded deadline"
}
function test_repo_remote_exec_file_upload() {
# Test that repository_ctx.execute accepts arguments of type label and can upload files and
# execute them remotely.
cat > BUILD <<'EOF'
exports_files(["cmd.sh", "hello.txt"])
EOF
cat > cmd.sh <<'EOF'
#!/bin/sh
cat $1
EOF
chmod +x cmd.sh
echo "hello world" > hello.txt
cat > test.bzl <<'EOF'
def _impl(ctx):
script = Label("//:cmd.sh")
file = Label("//:hello.txt")
res = ctx.execute([script, file])
if res.return_code != 0:
fail("Return code 0 expected, but was " + res.return_code)
if res.stdout.strip() != "hello world":
fail("Stdout 'hello world' expected, but was '" + res.stdout + "'");
ctx.file("BUILD")
remote_foo_configure = repository_rule(
implementation = _impl,
remotable = True,
)
local_foo_configure = repository_rule(
implementation = _impl,
)
EOF
cat > WORKSPACE <<'EOF'
load("//:test.bzl", "remote_foo_configure", "local_foo_configure")
remote_foo_configure(
name = "remote_foo",
)
local_foo_configure(
name = "local_foo",
)
EOF
bazel fetch \
--remote_executor=grpc://localhost:${worker_port} \
--experimental_repo_remote_exec \
@remote_foo//:all
# '--expunge' is necessary in order to ensure that the repository is re-executed.
bazel clean --expunge
# Run on the host machine to test that the rule works for both local and remote execution.
# In particular, that arguments of type label are accepted when doing local execution.
bazel fetch \
--experimental_repo_remote_exec \
@remote_foo//:all
bazel clean --expunge
# Execute @local_foo which has the same implementation as @remote_foo but not the 'remotable'
# attribute. This tests that a non-remotable repo rule can also run a remotable implementation
# function.
bazel fetch \
--experimental_repo_remote_exec \
@local_foo//:all
}
function test_remote_cache_intermediate_outputs() {
# test that remote cache is hit when intermediate output is not executable
touch WORKSPACE
cat > BUILD <<'EOF'
genrule(
name = "dep",
srcs = [],
outs = ["dep"],
cmd = "echo 'dep' > $@",
)
genrule(
name = "test",
srcs = [":dep"],
outs = ["out"],
cmd = "cat $(SRCS) > $@",
)
EOF
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//:test >& $TEST_log || fail "Failed to build //:test"
bazel clean
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//:test >& $TEST_log || fail "Failed to build //:test"
expect_log "2 remote cache hit"
}
function test_exclusive_tag() {
# Test that the exclusive tag works with the remote cache.
mkdir -p a
cat > a/success.sh <<'EOF'
#!/bin/sh
exit 0
EOF
chmod 755 a/success.sh
cat > a/BUILD <<'EOF'
sh_test(
name = "success_test",
srcs = ["success.sh"],
tags = ["exclusive"],
)
EOF
bazel test \
--incompatible_exclusive_test_sandboxed \
--remote_cache=grpc://localhost:${worker_port} \
//a:success_test || fail "Failed to test //a:success_test"
bazel test \
--incompatible_exclusive_test_sandboxed \
--remote_cache=grpc://localhost:${worker_port} \
--nocache_test_results \
//a:success_test >& $TEST_log || fail "Failed to test //a:success_test"
expect_log "remote cache hit"
}
# TODO(alpha): Add a test that fails remote execution when remote worker
# supports sandbox.
# This test uses the flag experimental_split_coverage_postprocessing. Without
# the flag coverage won't work remotely. Without the flag, tests and coverage
# post-processing happen in the same spawn, but only the runfiles tree of the
# tests is made available to the spawn. The solution was not to merge the
# runfiles tree which could cause its own problems but to split both into
# different spawns. The reason why this only failed remotely and not locally was
# because the coverage post-processing tool escaped the sandbox to find its own
# runfiles. The error we would see here without the flag would be "Cannot find
# runfiles". See #4685.
function test_java_rbe_coverage_produces_report() {
mkdir -p java/factorial
JAVA_TOOLS_ZIP="released"
COVERAGE_GENERATOR_DIR="released"
cd java/factorial
cat > BUILD <<'EOF'
java_library(
name = "fact",
srcs = ["Factorial.java"],
)
java_test(
name = "fact-test",
size = "small",
srcs = ["FactorialTest.java"],
test_class = "factorial.FactorialTest",
deps = [
":fact",
],
)
EOF
cat > Factorial.java <<'EOF'
package factorial;
public class Factorial {
public static int factorial(int x) {
return x <= 0 ? 1 : x * factorial(x-1);
}
}
EOF
cat > FactorialTest.java <<'EOF'
package factorial;
import static org.junit.Assert.*;
import org.junit.Test;
public class FactorialTest {
@Test
public void testFactorialOfZeroIsOne() throws Exception {
assertEquals(Factorial.factorial(3),6);
}
}
EOF
cd ../..
cat $(rlocation io_bazel/src/test/shell/bazel/testdata/jdk_http_archives) >> WORKSPACE
bazel coverage \
--test_output=all \
--experimental_fetch_all_coverage_outputs \
--experimental_split_coverage_postprocessing \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--instrumentation_filter=//java/factorial \
//java/factorial:fact-test >& $TEST_log || fail "Shouldn't fail"
local expected_result="SF:java/factorial/Factorial.java
FN:3,factorial/Factorial::<init> ()V
FN:5,factorial/Factorial::factorial (I)I
FNDA:0,factorial/Factorial::<init> ()V
FNDA:1,factorial/Factorial::factorial (I)I
FNF:2
FNH:1
BRDA:5,0,0,1
BRDA:5,0,1,1
BRF:2
BRH:2
DA:3,0
DA:5,1
LH:1
LF:2
end_of_record"
assert_equals "$expected_result" "$(cat bazel-testlogs/java/factorial/fact-test/coverage.dat)"
}
function generate_empty_tree_artifact_as_inputs() {
touch WORKSPACE
mkdir -p pkg
cat > pkg/def.bzl <<'EOF'
def _r(ctx):
empty_d = ctx.actions.declare_directory("%s/empty_dir" % ctx.label.name)
ctx.actions.run_shell(
outputs = [empty_d],
command = "mkdir -p %s" % empty_d.path,
)
f = ctx.actions.declare_file("%s/file" % ctx.label.name)
ctx.actions.run_shell(
inputs = [empty_d],
outputs = [f],
command = "touch %s && cd %s && pwd" % (f.path, empty_d.path),
)
return [DefaultInfo(files = depset([f]))]
r = rule(implementation = _r)
EOF
cat > pkg/BUILD <<'EOF'
load(":def.bzl", "r")
r(name = "a")
EOF
}
function test_empty_tree_artifact_as_inputs() {
# Test that when an empty tree artifact is the input, an empty directory is
# created in the remote executor for action to read.
generate_empty_tree_artifact_as_inputs
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
//pkg:a &>$TEST_log || fail "expected build to succeed"
bazel clean --expunge
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--experimental_remote_merkle_tree_cache \
//pkg:a &>$TEST_log || fail "expected build to succeed with Merkle tree cache"
bazel clean --expunge
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--experimental_sibling_repository_layout \
//pkg:a &>$TEST_log || fail "expected build to succeed with sibling repository layout"
}
function test_empty_tree_artifact_as_inputs_remote_cache() {
# Test that when empty tree artifact works for remote cache.
generate_empty_tree_artifact_as_inputs
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//pkg:a &>$TEST_log || fail "expected build to succeed"
bazel clean
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//pkg:a &>$TEST_log || fail "expected build to succeed"
expect_log "remote cache hit"
}
function generate_tree_artifact_output() {
touch WORKSPACE
mkdir -p pkg
cat > pkg/def.bzl <<'EOF'
def _r(ctx):
empty_dir = ctx.actions.declare_directory("%s/empty_dir" % ctx.label.name)
ctx.actions.run_shell(
outputs = [empty_dir],
command = "cd %s && pwd" % empty_dir.path,
)
non_empty_dir = ctx.actions.declare_directory("%s/non_empty_dir" % ctx.label.name)
ctx.actions.run_shell(
outputs = [non_empty_dir],
command = "cd %s && pwd && touch out" % non_empty_dir.path,
)
return [DefaultInfo(files = depset([empty_dir, non_empty_dir]))]
r = rule(implementation = _r)
EOF
cat > pkg/BUILD <<'EOF'
load(":def.bzl", "r")
r(name = "a")
EOF
}
function test_create_tree_artifact_outputs() {
# Test that if a tree artifact is declared as an input, then the corresponding
# empty directory is created before the action executes remotely.
generate_tree_artifact_output
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
//pkg:a &>$TEST_log || fail "expected build to succeed"
[[ -f bazel-bin/pkg/a/non_empty_dir/out ]] || fail "expected tree artifact to contain a file"
[[ -d bazel-bin/pkg/a/empty_dir ]] || fail "expected directory to exist"
bazel clean --expunge
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--experimental_remote_merkle_tree_cache \
//pkg:a &>$TEST_log || fail "expected build to succeed with Merkle tree cache"
[[ -f bazel-bin/pkg/a/non_empty_dir/out ]] || fail "expected tree artifact to contain a file"
[[ -d bazel-bin/pkg/a/empty_dir ]] || fail "expected directory to exist"
bazel clean --expunge
bazel build \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--experimental_sibling_repository_layout \
//pkg:a &>$TEST_log || fail "expected build to succeed with sibling repository layout"
[[ -f bazel-bin/pkg/a/non_empty_dir/out ]] || fail "expected tree artifact to contain a file"
[[ -d bazel-bin/pkg/a/empty_dir ]] || fail "expected directory to exist"
}
function test_create_tree_artifact_outputs_remote_cache() {
# Test that implicitly created empty directories corresponding to empty tree
# artifacts outputs are correctly cached in the remote cache.
generate_tree_artifact_output
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//pkg:a &>$TEST_log || fail "expected build to succeed"
bazel clean
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//pkg:a &>$TEST_log || fail "expected build to succeed"
expect_log "2 remote cache hit"
[[ -f bazel-bin/pkg/a/non_empty_dir/out ]] || fail "expected tree artifact to contain a file"
[[ -d bazel-bin/pkg/a/empty_dir ]] || fail "expected directory to exist"
}
# Runs coverage with `cc_test` and RE then checks the coverage file is returned.
# Older versions of gcov are not supported with bazel coverage and so will be skipped.
# See the above `test_java_rbe_coverage_produces_report` for more information.
function test_cc_rbe_coverage_produces_report() {
if [[ "$PLATFORM" == "darwin" ]]; then
# TODO(b/37355380): This test is disabled due to RemoteWorker not supporting
# setting SDKROOT and DEVELOPER_DIR appropriately, as is required of
# action executors in order to select the appropriate Xcode toolchain.
return 0
fi
# Check to see if intermediate files are supported, otherwise skip.
gcov --help | grep "\-i," || return 0
local test_dir="a/cc/coverage_test"
mkdir -p $test_dir
cat > "$test_dir"/BUILD <<'EOF'
package(default_visibility = ["//visibility:public"])
cc_library(
name = "hello-lib",
srcs = ["hello-lib.cc"],
hdrs = ["hello-lib.h"],
)
cc_binary(
name = "hello-world",
srcs = ["hello-world.cc"],
deps = [":hello-lib"],
)
cc_test(
name = "hello-test",
srcs = ["hello-world.cc"],
deps = [":hello-lib"],
)
EOF
cat > "$test_dir"/hello-lib.cc <<'EOF'
#include "hello-lib.h"
#include <iostream>
using std::cout;
using std::endl;
using std::string;
namespace hello {
HelloLib::HelloLib(const string& greeting) : greeting_(new string(greeting)) {
}
void HelloLib::greet(const string& thing) {
cout << *greeting_ << " " << thing << endl;
}
} // namespace hello
EOF
cat > "$test_dir"/hello-lib.h <<'EOF'
#ifndef HELLO_LIB_H_
#define HELLO_LIB_H_
#include <string>
#include <memory>
namespace hello {
class HelloLib {
public:
explicit HelloLib(const std::string &greeting);
void greet(const std::string &thing);
private:
std::unique_ptr<const std::string> greeting_;
};
} // namespace hello
#endif // HELLO_LIB_H_
EOF
cat > "$test_dir"/hello-world.cc <<'EOF'
#include "hello-lib.h"
#include <string>
using hello::HelloLib;
using std::string;
int main(int argc, char** argv) {
HelloLib lib("Hello");
string thing = "world";
if (argc > 1) {
thing = argv[1];
}
lib.greet(thing);
return 0;
}
EOF
bazel coverage \
--test_output=all \
--experimental_fetch_all_coverage_outputs \
--experimental_split_coverage_postprocessing \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
//"$test_dir":hello-test >& $TEST_log \
|| fail "Failed to run coverage for cc_test"
# Different gcov versions generate different outputs.
# Simply check if this is empty or not.
if [[ ! -s bazel-testlogs/a/cc/coverage_test/hello-test/coverage.dat ]]; then
echo "Coverage is empty. Failing now."
return 1
fi
}
function test_grpc_connection_errors_are_propagated() {
# Test that errors when creating grpc connection are propagated instead of crashing Bazel.
# https://github.com/bazelbuild/bazel/issues/13724
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
)
EOF
bazel build \
--remote_executor=grpcs://localhost:${worker_port} \
--tls_certificate=/nope \
//a:foo >& $TEST_log && fail "Expected to fail" || true
expect_log "ERROR: Failed to query remote execution capabilities: Failed to init TLS infrastructure using '/nope' as root certificate: File does not contain valid certificates: /nope"
}
function test_async_upload_works_for_flaky_tests() {
mkdir -p a
cat > a/BUILD <<EOF
sh_test(
name = "test",
srcs = ["test.sh"],
)
genrule(
name = "foo",
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
)
EOF
cat > a/test.sh <<EOF
#!/bin/sh
echo "it always fails"
exit 1
EOF
chmod +x a/test.sh
# Check the error message when failed to upload
bazel build --remote_cache=http://nonexistent.example.org //a:foo >& $TEST_log || fail "Failed to build"
expect_log "WARNING: Remote Cache:"
bazel test \
--remote_cache=grpc://localhost:${worker_port} \
--experimental_remote_cache_async \
--flaky_test_attempts=2 \
//a:test >& $TEST_log && fail "expected failure" || true
expect_not_log "WARNING: Remote Cache:"
}
function test_uploader_respect_no_cache() {
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
tags = ["no-cache"],
)
EOF
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--incompatible_remote_build_event_upload_respect_no_cache \
--build_event_json_file=bep.json \
//a:foo >& $TEST_log || fail "Failed to build"
cat bep.json > $TEST_log
expect_not_log "a:foo.*bytestream://" || fail "local files are converted"
expect_log "command.profile.gz.*bytestream://" || fail "should upload profile data"
}
function test_uploader_respect_no_cache_minimal() {
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
tags = ["no-cache"],
)
EOF
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--remote_download_minimal \
--incompatible_remote_build_event_upload_respect_no_cache \
--build_event_json_file=bep.json \
//a:foo >& $TEST_log || fail "Failed to build"
cat bep.json > $TEST_log
expect_not_log "a:foo.*bytestream://" || fail "local files are converted"
expect_log "command.profile.gz.*bytestream://" || fail "should upload profile data"
}
function test_uploader_alias_action_respect_no_cache() {
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
tags = ["no-cache"],
)
alias(
name = 'foo-alias',
actual = '//a:foo',
)
EOF
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--incompatible_remote_build_event_upload_respect_no_cache \
--build_event_json_file=bep.json \
//a:foo-alias >& $TEST_log || fail "Failed to build"
cat bep.json > $TEST_log
expect_not_log "a:foo.*bytestream://"
expect_log "command.profile.gz.*bytestream://"
}
function test_uploader_respect_no_cache_trees() {
mkdir -p a
cat > a/output_dir.bzl <<'EOF'
def _gen_output_dir_impl(ctx):
output_dir = ctx.actions.declare_directory(ctx.attr.outdir)
ctx.actions.run_shell(
outputs = [output_dir],
inputs = [],
command = """
mkdir -p $1/sub; \
index=0; while ((index<10)); do echo $index >$1/$index.txt; index=$(($index+1)); done
echo "Shuffle, duffle, muzzle, muff" > $1/sub/bar
""",
arguments = [output_dir.path],
execution_requirements = {"no-cache": ""},
)
return [
DefaultInfo(files = depset(direct = [output_dir])),
]
gen_output_dir = rule(
implementation = _gen_output_dir_impl,
attrs = {
"outdir": attr.string(mandatory = True),
},
)
EOF
cat > a/BUILD <<EOF
load(":output_dir.bzl", "gen_output_dir")
gen_output_dir(
name = "foo",
outdir = "dir",
)
EOF
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--incompatible_remote_build_event_upload_respect_no_cache \
--build_event_json_file=bep.json \
//a:foo >& $TEST_log || fail "Failed to build"
cat bep.json > $TEST_log
expect_not_log "a:foo.*bytestream://" || fail "local tree files are converted"
expect_not_log "a/dir/.*bytestream://" || fail "local tree files are converted"
expect_log "command.profile.gz.*bytestream://" || fail "should upload profile data"
}
function test_uploader_respect_no_upload_results() {
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
)
EOF
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--remote_upload_local_results=false \
--incompatible_remote_build_event_upload_respect_no_cache \
--build_event_json_file=bep.json \
//a:foo >& $TEST_log || fail "Failed to build"
cat bep.json > $TEST_log
expect_not_log "a:foo.*bytestream://" || fail "local files are converted"
expect_log "command.profile.gz.*bytestream://" || fail "should upload profile data"
}
function test_uploader_respect_no_upload_results_combined_cache() {
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
)
EOF
cache_dir=$(mktemp -d)
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--disk_cache=$cache_dir \
--remote_upload_local_results=false \
--incompatible_remote_build_event_upload_respect_no_cache \
--build_event_json_file=bep.json \
//a:foo >& $TEST_log || fail "Failed to build"
cat bep.json > $TEST_log
expect_not_log "a:foo.*bytestream://" || fail "local files are converted"
expect_log "command.profile.gz.*bytestream://" || fail "should upload profile data"
remote_cas_files="$(count_remote_cas_files)"
[[ "$remote_cas_files" == 1 ]] || fail "Expected 1 remote cas entries, not $remote_cas_files"
}
function test_remote_exec_convert_remote_file() {
mkdir -p a
cat > a/BUILD <<'EOF'
sh_test(
name = "test",
srcs = ["test.sh"],
)
EOF
cat > a/test.sh <<'EOF'
#!/bin/bash
set -e
echo \"foo\"
exit 0
EOF
chmod 755 a/test.sh
bazel test \
--remote_executor=grpc://localhost:${worker_port} \
--build_event_json_file=bep.json \
--noremote_upload_local_results \
--incompatible_remote_build_event_upload_respect_no_cache \
//a:test >& $TEST_log || "Failed to test //a:test"
cat bep.json > $TEST_log
expect_not_log "test\.log.*file://" || fail "remote files generated in remote execution are not converted"
expect_log "test\.log.*bytestream://" || fail "remote files generated in remote execution are not converted"
}
function test_uploader_ignore_disk_cache_of_combined_cache() {
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
tags = ["no-cache"],
)
EOF
cache_dir=$(mktemp -d)
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--disk_cache=$cache_dir \
--incompatible_remote_build_event_upload_respect_no_cache \
--build_event_json_file=bep.json \
//a:foo >& $TEST_log || fail "Failed to build"
cat bep.json > $TEST_log
expect_not_log "a:foo.*bytestream://" || fail "local files are converted"
expect_log "command.profile.gz.*bytestream://" || fail "should upload profile data"
disk_cas_files="$(count_disk_cas_files $cache_dir)"
[[ "$disk_cas_files" == 0 ]] || fail "Expected 0 disk cas entries, not $disk_cas_files"
}
function test_uploader_incompatible_remote_results_ignore_disk() {
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo \"foo bar\" > \$@",
tags = ["no-remote"],
)
EOF
cache_dir=$(mktemp -d)
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
--disk_cache=$cache_dir \
--incompatible_remote_build_event_upload_respect_no_cache \
--incompatible_remote_results_ignore_disk \
--build_event_json_file=bep.json \
//a:foo >& $TEST_log || fail "Failed to build"
cat bep.json > $TEST_log
expect_not_log "a:foo.*bytestream://" || fail "local files are converted"
expect_log "command.profile.gz.*bytestream://" || fail "should upload profile data"
disk_cas_files="$(count_disk_cas_files $cache_dir)"
# foo.txt, stdout and stderr for action 'foo'
[[ "$disk_cas_files" == 3 ]] || fail "Expected 3 disk cas entries, not $disk_cas_files"
}
function test_missing_outputs_dont_upload_action_result() {
# Test that if declared outputs are not created, even the exit code of action
# is 0, we treat this as failed and don't upload action result.
# See https://github.com/bazelbuild/bazel/issues/14543.
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "echo foo",
)
EOF
bazel build \
--remote_cache=grpc://localhost:${worker_port} \
//a:foo >& $TEST_log && fail "Should failed to build"
remote_cas_files="$(count_remote_cas_files)"
[[ "$remote_cas_files" == 0 ]] || fail "Expected 0 remote cas entries, not $remote_cas_files"
remote_ac_files="$(count_remote_ac_files)"
[[ "$remote_ac_files" == 0 ]] || fail "Expected 0 remote action cache entries, not $remote_ac_files"
}
function test_failed_action_dont_check_declared_outputs() {
# Test that if action failed, outputs are not checked
mkdir -p a
cat > a/BUILD <<EOF
genrule(
name = 'foo',
outs = ["foo.txt"],
cmd = "exit 1",
)
EOF
bazel build \
--remote_executor=grpc://localhost:${worker_port} \
//a:foo >& $TEST_log && fail "Should failed to build"
expect_log "Executing genrule .* failed: (Exit 1):"
}
function test_local_test_execution_with_disk_cache() {
# Tests that the wall time for a locally executed test is correctly cached.
# If not, the generate-xml.sh action, which embeds the wall time, will be
# considered stale on a cache hit.
# Regression test for https://github.com/bazelbuild/bazel/issues/14426.
mkdir -p a
cat > a/BUILD <<EOF
sh_test(
name = 'test',
srcs = ['test.sh'],
)
EOF
cat > a/test.sh <<EOF
sleep 1
EOF
chmod +x a/test.sh
CACHEDIR=$(mktemp -d)
bazel test \
--disk_cache=$CACHEDIR \
//a:test >& $TEST_log || "Failed to build //a:test"
expect_log "5 processes: 3 internal, 2 .*-sandbox"
bazel clean
bazel test \
--disk_cache=$CACHEDIR \
//a:test >& $TEST_log || "Failed to build //a:test"
expect_log "5 processes: 2 disk cache hit, 3 internal"
}
# Bazel assumes that non-ASCII characters in file contents (and, in
# non-Windows systems, file paths) are UTF-8, but stores them internally by
# parsing the raw UTF-8 bytes as if they were ISO-8859-1 characters.
#
# This test verifies that the inverse transformation is applied when sending
# these inputs through the remote execution protocol. The Protobuf libraries
# for Java assume that `String` values are encoded in UTF-16, and passing
# raw bytes will cause double-encoding.
function test_unicode() {
# The in-tree remote execution worker only supports non-ASCII paths when
# running in a UTF-8 locale.
if ! "$is_windows"; then
if ! has_utf8_locale; then
echo "Skipping test due to lack of UTF-8 locale."
echo "Available locales:"
locale -a
return
fi
fi
# Restart the remote execution worker with LC_ALL set.
tear_down
LC_ALL=en_US.UTF-8 set_up
# The test inputs contain both globbed and unglobbed input paths, to verify
# consistent behavior on Windows where file paths are Unicode but Starlark
# strings are UTF-8.
cat > BUILD <<EOF
load("//:rules.bzl", "test_unicode")
test_unicode(
name = "test_unicode",
inputs = glob(["inputs/**/A_*"]) + [
"inputs/入力/B_🌱.txt",
],
)
EOF
cat > rules.bzl <<'EOF'
def _test_unicode(ctx):
out_dir = ctx.actions.declare_directory(ctx.attr.name + "_outs/出力/🌱.d")
out_file = ctx.actions.declare_file(ctx.attr.name + "_outs/出力/🌱.txt")
out_report = ctx.actions.declare_file(ctx.attr.name + "_report.txt")
ctx.actions.run_shell(
inputs = ctx.files.inputs,
outputs = [out_dir, out_file, out_report],
command = """
set -eu
report="$3"
touch "${report}"
echo '[input tree]' >> "${report}"
find inputs | sort >> "${report}"
echo '' >> "${report}"
echo '[input file A]' >> "${report}"
cat $'inputs/\\xe5\\x85\\xa5\\xe5\\x8a\\x9b/A_\\xf0\\x9f\\x8c\\xb1.txt' >> "${report}"
echo '' >> "${report}"
echo '[input file B]' >> "${report}"
cat $'inputs/\\xe5\\x85\\xa5\\xe5\\x8a\\x9b/B_\\xf0\\x9f\\x8c\\xb1.txt' >> "${report}"
echo '' >> "${report}"
echo '[environment]' >> "${report}"
env | grep -v BASH_EXECUTION_STRING | grep TEST_UNICODE_ >> "${report}"
echo '' >> "${report}"
mkdir -p "$1"
echo 'output dir content' > "$1"/dir_content.txt
echo 'output file content' > "$2"
""",
arguments = [out_dir.path, out_file.path, out_report.path],
env = {"TEST_UNICODE_🌱": "🌱"},
)
return DefaultInfo(files=depset([out_dir, out_file, out_report]))
test_unicode = rule(
implementation = _test_unicode,
attrs = {
"inputs": attr.label_list(allow_files = True),
},
)
EOF
# inputs/入力/A_🌱.txt and inputs/入力/B_🌱.txt
mkdir -p $'inputs/\xe5\x85\xa5\xe5\x8a\x9b'
echo 'input content A' > $'inputs/\xe5\x85\xa5\xe5\x8a\x9b/A_\xf0\x9f\x8c\xb1.txt'
echo 'input content B' > $'inputs/\xe5\x85\xa5\xe5\x8a\x9b/B_\xf0\x9f\x8c\xb1.txt'
# On UNIX platforms, Bazel assumes that file paths are encoded in UTF-8. The
# system must have either an ISO-8859-1 or UTF-8 locale available so that
# Bazel can read the original bytes of the file path.
#
# If no ISO-8859-1 locale is available, the JVM might fall back to US-ASCII
# rather than trying UTF-8. Setting `LC_ALL=en_US.UTF-8` prevents this.
bazel shutdown
LC_ALL=en_US.UTF-8 bazel build \
--remote_executor=grpc://localhost:${worker_port} \
//:test_unicode >& $TEST_log \
|| fail "Failed to build //:test_unicode with remote execution"
expect_log "2 processes: 1 internal, 1 remote."
# Don't leak LC_ALL into other tests.
bazel shutdown
# Expect action outputs with correct structure and content.
mkdir -p ${TEST_TMPDIR}/$'test_unicode_outs/\xe5\x87\xba\xe5\x8a\x9b/\xf0\x9f\x8c\xb1.d'
cat > ${TEST_TMPDIR}/$'test_unicode_outs/\xe5\x87\xba\xe5\x8a\x9b/\xf0\x9f\x8c\xb1.d/dir_content.txt' <<EOF
output dir content
EOF
cat > ${TEST_TMPDIR}/$'test_unicode_outs/\xe5\x87\xba\xe5\x8a\x9b/\xf0\x9f\x8c\xb1.txt' <<EOF
output file content
EOF
diff -r bazel-bin/test_unicode_outs ${TEST_TMPDIR}/test_unicode_outs \
|| fail "Remote execution generated different result"
cat > ${TEST_TMPDIR}/test_report_expected <<EOF
[input tree]
inputs
inputs/入力
inputs/入力/A_🌱.txt
inputs/入力/B_🌱.txt
[input file A]
input content A
[input file B]
input content B
[environment]
TEST_UNICODE_🌱=🌱
EOF
diff bazel-bin/test_unicode_report.txt ${TEST_TMPDIR}/test_report_expected \
|| fail "Remote execution generated different result"
}
function test_external_cc_test() {
if [[ "$PLATFORM" == "darwin" ]]; then
# TODO(b/37355380): This test is disabled due to RemoteWorker not supporting
# setting SDKROOT and DEVELOPER_DIR appropriately, as is required of
# action executors in order to select the appropriate Xcode toolchain.
return 0
fi
cat >> WORKSPACE <<'EOF'
local_repository(
name = "other_repo",
path = "other_repo",
)
EOF
mkdir -p other_repo
touch other_repo/WORKSPACE
mkdir -p other_repo/lib
cat > other_repo/lib/BUILD <<'EOF'
cc_library(
name = "lib",
srcs = ["lib.cpp"],
hdrs = ["lib.h"],
visibility = ["//visibility:public"],
)
EOF
cat > other_repo/lib/lib.h <<'EOF'
void print_greeting();
EOF
cat > other_repo/lib/lib.cpp <<'EOF'
#include <cstdio>
void print_greeting() {
printf("Hello, world!\n");
}
EOF
mkdir -p other_repo/test
cat > other_repo/test/BUILD <<'EOF'
cc_test(
name = "test",
srcs = ["test.cpp"],
deps = ["//lib"],
)
EOF
cat > other_repo/test/test.cpp <<'EOF'
#include "lib/lib.h"
int main() {
print_greeting();
}
EOF
bazel test \
--test_output=errors \
--remote_executor=grpc://localhost:${worker_port} \
@other_repo//test >& $TEST_log || fail "Test should pass"
}
function test_unresolved_symlink_input() {
mkdir -p symlink
touch symlink/BUILD
cat > symlink/symlink.bzl <<EOF
def _dangling_symlink_impl(ctx):
symlink = ctx.actions.declare_symlink(ctx.label.name)
ctx.actions.symlink(
output = symlink,
target_path = ctx.attr.link_target,
)
return DefaultInfo(files = depset([symlink]))
dangling_symlink = rule(
implementation = _dangling_symlink_impl,
attrs = {"link_target": attr.string()},
)
EOF
mkdir -p pkg
cat > pkg/BUILD <<'EOF'
load("//symlink:symlink.bzl", "dangling_symlink")
dangling_symlink(name="a", link_target="non/existent")
genrule(
name = "b",
srcs = [":a"],
outs = ["b.txt"],
cmd = "readlink $(location :a) > $@",
)
EOF
bazel \
--windows_enable_symlinks \
build \
--experimental_allow_unresolved_symlinks \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
//pkg:b &>$TEST_log || fail "expected build to succeed"
# Using regex because the symlink is always absolute. https://github.com/bazelbuild/bazel/issues/14224
[[ $(cat bazel-bin/pkg/b.txt) =~ .*/non/existent ]] || fail "expected symlink target to be non/existent"
bazel clean --expunge
bazel \
--windows_enable_symlinks \
build \
--experimental_allow_unresolved_symlinks \
--spawn_strategy=remote \
--remote_executor=grpc://localhost:${worker_port} \
--experimental_remote_merkle_tree_cache \
//pkg:b &>$TEST_log || fail "expected build to succeed with Merkle tree cache"
# Using regex because the symlink is always absolute. https://github.com/bazelbuild/bazel/issues/14224
[[ $(cat bazel-bin/pkg/b.txt) =~ .*/non/existent ]] || fail "expected symlink target to be non/existent"
}
run_suite "Remote execution and remote cache tests"