src/test/shell/integration/runfiles_test.sh

#!/bin/bash
#
# Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# An end-to-end test that Bazel produces runfiles trees as expected.

# --- begin runfiles.bash initialization ---
# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash).
set -euo pipefail
if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
  if [[ -f "$0.runfiles_manifest" ]]; then
    export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
  elif [[ -f "$0.runfiles/MANIFEST" ]]; then
    export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
  elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
    export RUNFILES_DIR="$0.runfiles"
  fi
fi
if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
  source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
  source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
            "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
else
  echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
  exit 1
fi
# --- end runfiles.bash initialization ---

source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \
  || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
source "$(rlocation "io_bazel/src/test/shell/integration/runfiles_test_utils.sh")" \
  || { echo "runfiles_test_utils.sh not found!" >&2; exit 1; }

case "$(uname -s | tr [:upper:] [:lower:])" in
msys*|mingw*|cygwin*)
  declare -r is_windows=true
  ;;
*)
  declare -r is_windows=false
  ;;
esac

# We disable Python toolchains in EXTRA_BUILD_FLAGS because it throws off the
# counts and manifest checks in test_foo_runfiles.
# TODO(#8169): Update this test and remove the toolchain opt-out.
if "$is_windows"; then
  export MSYS_NO_PATHCONV=1
  export MSYS2_ARG_CONV_EXCL="*"
  export EXT=".exe"
  export EXTRA_BUILD_FLAGS="--incompatible_use_python_toolchains=false \
--enable_runfiles --build_python_zip=0"
else
  export EXT=""
  export EXTRA_BUILD_FLAGS="--incompatible_use_python_toolchains=false"
fi

#### SETUP #############################################################

set -e

disable_bzlmod

function create_pkg() {
  local -r pkg=$1
  mkdir -p $pkg
  cd $pkg

  mkdir -p a/b c/d e/f/g x/y
  touch py.py a/b/no_module.py c/d/one_module.py c/__init__.py e/f/g/ignored.txt x/y/z.sh
  chmod +x x/y/z.sh

  cd ..
  touch __init__.py
}

# This is basically a cross-platform version of `find -printf '%n %y %Y'`.
# i.e. recursively print paths, their raw file type, and for symbolic links,
# the type of file the link points to.
# Macs don't support `find -printf`, and stat, readlink etc all have different
# args and format specifiers. Basic bash works fine, though.
function recursive_path_info() {
  for path in $(find "$1" | sort); do
    if [[ -L "$path" ]]; then
      actual_type=symlink
    else
      actual_type=regular
    fi
    if [[ -f "$path" ]]; then
      effective_type=file
    elif [[ -d "$path" ]]; then
      effective_type="$actual_type dir"
    else
      # The various special file types shouldn't occur in practice, so just
      # call them unknown
      effective_type=unknown
    fi
    echo "$path $effective_type"
  done
}

#### TESTS #############################################################

function test_hidden() {
  local -r pkg=$FUNCNAME

  mkdir -p "$pkg/e/f/g"
  touch "$pkg/e/f/g/hidden.txt"
  cat > "$pkg/defs.bzl" << EOF
def _obscured_impl(ctx):
    executable = ctx.actions.declare_file(ctx.label.name)
    ctx.actions.write(executable, "# nop")
    return [DefaultInfo(
        executable = executable,
        runfiles = ctx.runfiles(files = ctx.files.data),
    )]

obscured = rule(
    implementation = _obscured_impl,
    attrs = {"data": attr.label_list(allow_files = True)},
    # Must be executable to trigger the obscured runfile check
    executable = True,
)
EOF

  cat > $pkg/BUILD << EOF
load(":defs.bzl", "obscured")
obscured(name="bin", data=["e/f", "e/f/g/hidden.txt"])
genrule(name = "hidden",
        outs = [ "e/f/g/hidden.txt" ],
        cmd = "touch \$@")
EOF
  bazel build $pkg:bin $EXTRA_BUILD_FLAGS >&$TEST_log 2>&1 || fail "build failed"

  # we get a warning that hidden.txt is inaccessible
  expect_log_once "${pkg}/e/f/g/hidden.txt obscured by ${pkg}/e/f "
}

function test_foo_runfiles() {
  local -r pkg=$FUNCNAME
  create_pkg $pkg
cat > BUILD << EOF
load("@rules_python//python:py_library.bzl", "py_library")

py_library(name = "root",
           srcs = ["__init__.py"],
           visibility = ["//visibility:public"])
EOF
cat > $pkg/BUILD << EOF
load("@rules_python//python:py_binary.bzl", "py_binary")

sh_binary(name = "foo",
          srcs = [ "x/y/z.sh" ],
          data = [ ":py",
                   "e/f" ])
py_binary(name = "py",
          srcs = [ "py.py",
                   "a/b/no_module.py",
                   "c/d/one_module.py",
                   "c/__init__.py",
                 ],
          data = ["e/f/g/ignored.txt"],
          deps = ["//:root"])
EOF
  bazel build $pkg:foo $EXTRA_BUILD_FLAGS >&$TEST_log || fail "build failed"
  workspace_root=$PWD

  cd ${PRODUCT_NAME}-bin/$pkg/foo${EXT}.runfiles

  # workaround until we use assert/fail macros in the tests below
  touch $TEST_TMPDIR/__fail

  # output manifest exists and is non-empty
  test    -f MANIFEST
  test    -s MANIFEST

  cd ${WORKSPACE_NAME}


  cd $pkg

  # these are real empty files
  test \! -s a/__init__.py
  test \! -s a/b/__init__.py
  test \! -s c/d/__init__.py
  test \! -s __init__.py
  cd ..

  # These are basically tuples of (path filetype)
  expected="
. regular dir
./__init__.py file
./test_foo_runfiles regular dir
./test_foo_runfiles/__init__.py file
./test_foo_runfiles/a regular dir
./test_foo_runfiles/a/__init__.py file
./test_foo_runfiles/a/b regular dir
./test_foo_runfiles/a/b/__init__.py file
./test_foo_runfiles/a/b/no_module.py file
./test_foo_runfiles/c regular dir
./test_foo_runfiles/c/__init__.py file
./test_foo_runfiles/c/d regular dir
./test_foo_runfiles/c/d/__init__.py file
./test_foo_runfiles/c/d/one_module.py file
./test_foo_runfiles/e regular dir
./test_foo_runfiles/e/f symlink dir
./test_foo_runfiles/foo file
./test_foo_runfiles/py file
./test_foo_runfiles/py.py file
./test_foo_runfiles/x regular dir
./test_foo_runfiles/x/y regular dir
./test_foo_runfiles/x/y/z.sh file
"
  expected="$expected$(get_python_runtime_runfiles)"

  # For shell binary and python binary, we build both `bin` and `bin.exe`,
  # but on Linux we only build `bin`.
  if "$is_windows"; then
    expected="${expected}
./test_foo_runfiles/py.exe file
./test_foo_runfiles/foo.exe file
"
  fi

  # Sort and delete empty lines. This makes it easier to append to the
  # expected string and not have to worry about stray newlines from shell
  # commands and quoting.
  expected=$(sort <<<"$expected" | sed '/^$/d')
  actual=$(recursive_path_info .)
  assert_equals "$expected" "$actual"

  # The manifest only records files and symlinks, not real directories
  expected="$expected$(get_repo_mapping_manifest_file)"
  expected_manifest_size=$(echo "$expected" | grep -v ' regular dir' | wc -l)
  actual_manifest_size=$(wc -l < ../MANIFEST)
  assert_equals $expected_manifest_size $actual_manifest_size

  # that accounts for everything
  cd ..

  for i in $(find ${WORKSPACE_NAME} \! -type d); do
    target="$(readlink "$i" || true)"
    if [[ -z "$target" ]]; then
      echo "$i " >> ${TEST_TMPDIR}/MANIFEST2
    else
      if "$is_windows"; then
        echo "$i $(cygpath -m $target)" >> ${TEST_TMPDIR}/MANIFEST2
      else
        echo "$i $target" >> ${TEST_TMPDIR}/MANIFEST2
      fi
    fi
  done

  # Add the repo mapping manifest entry for Bazel.
  if [[ "$PRODUCT_NAME" == "bazel" ]]; then
    repo_mapping="_repo_mapping"
    repo_mapping_target="$(readlink "$repo_mapping")"
    if "$is_windows"; then
      repo_mapping_target="$(cygpath -m $repo_mapping_target)"
    fi
    echo "$repo_mapping $repo_mapping_target" >> ${TEST_TMPDIR}/MANIFEST2
  fi

  sort MANIFEST > ${TEST_TMPDIR}/MANIFEST_sorted
  sort ${TEST_TMPDIR}/MANIFEST2 > ${TEST_TMPDIR}/MANIFEST2_sorted
  diff -u ${TEST_TMPDIR}/MANIFEST_sorted ${TEST_TMPDIR}/MANIFEST2_sorted

  # Rebuild the same target with a new dependency.
  cd "$workspace_root"
cat > $pkg/BUILD << EOF
sh_binary(name = "foo",
          srcs = [ "x/y/z.sh" ],
          data = [ "e/f" ])
EOF
  bazel build $pkg:foo $EXTRA_BUILD_FLAGS >&$TEST_log || fail "build failed"

  cd ${PRODUCT_NAME}-bin/$pkg/foo${EXT}.runfiles

  # workaround until we use assert/fail macros in the tests below
  touch $TEST_TMPDIR/__fail

  # output manifest exists and is non-empty
  test    -f MANIFEST
  test    -s MANIFEST

  cd ${WORKSPACE_NAME}

  # these are real directories
  test \! -L $pkg
  test    -d $pkg

  # these directory should not exist anymore
  test \! -e a
  test \! -e c

  cd $pkg
  test \! -L e
  test    -d e
  test \! -L x
  test    -d x
  test \! -L x/y
  test    -d x/y

  # these are symlinks to the source tree
  test    -L foo
  test    -L x/y/z.sh
  test    -L e/f
  test    -d e/f

  # that accounts for everything
  cd ../..
  # For shell binary, we build both `bin` and `bin.exe`, but on Linux we only build `bin`
  # That's why we have one more symlink on Windows.
  if "$is_windows"; then
    assert_equals  4 $(find ${WORKSPACE_NAME} -type l | wc -l)
    assert_equals  0 $(find ${WORKSPACE_NAME} -type f | wc -l)
    assert_equals  5 $(find ${WORKSPACE_NAME} -type d | wc -l)
    assert_equals  9 $(find ${WORKSPACE_NAME} | wc -l)
    if [[ "$PRODUCT_NAME" == "bazel" ]]; then
      assert_equals  5 $(wc -l < MANIFEST)
    else
      assert_equals  4 $(wc -l < MANIFEST)
    fi
  else
    assert_equals  3 $(find ${WORKSPACE_NAME} -type l | wc -l)
    assert_equals  0 $(find ${WORKSPACE_NAME} -type f | wc -l)
    assert_equals  5 $(find ${WORKSPACE_NAME} -type d | wc -l)
    assert_equals  8 $(find ${WORKSPACE_NAME} | wc -l)
    if [[ "$PRODUCT_NAME" == "bazel" ]]; then
      assert_equals  4 $(wc -l < MANIFEST)
    else
      assert_equals  3 $(wc -l < MANIFEST)
    fi
  fi

  rm -f ${TEST_TMPDIR}/MANIFEST
  rm -f ${TEST_TMPDIR}/MANIFEST2
  for i in $(find ${WORKSPACE_NAME} \! -type d); do
    target="$(readlink "$i" || true)"
    if [[ -z "$target" ]]; then
      echo "$i " >> ${TEST_TMPDIR}/MANIFEST2
    else
      if "$is_windows"; then
        echo "$i $(cygpath -m $target)" >> ${TEST_TMPDIR}/MANIFEST2
      else
        echo "$i $target" >> ${TEST_TMPDIR}/MANIFEST2
      fi
    fi
  done

  # Add the repo mapping manifest entry for Bazel.
  if [[ "$PRODUCT_NAME" == "bazel" ]]; then
    repo_mapping="_repo_mapping"
    repo_mapping_target="$(readlink "$repo_mapping")"
    if "$is_windows"; then
      repo_mapping_target="$(cygpath -m $repo_mapping_target)"
    fi
    echo "$repo_mapping $repo_mapping_target" >> ${TEST_TMPDIR}/MANIFEST2
  fi

  sort MANIFEST > ${TEST_TMPDIR}/MANIFEST_sorted
  sort ${TEST_TMPDIR}/MANIFEST2 > ${TEST_TMPDIR}/MANIFEST2_sorted
  diff -u ${TEST_TMPDIR}/MANIFEST_sorted ${TEST_TMPDIR}/MANIFEST2_sorted
}

function test_workspace_name_change() {
  # TODO(b/174761497): Re-enable the test outside of Bazel.
  [[ "${PRODUCT_NAME}" != bazel ]] && return 0

  # Rewrite the workspace name but leave the rest of WORKSPACE alone.
  sed -ie 's,workspace(.*,workspace(name = "foo"),' WORKSPACE

  cat > BUILD <<EOF
cc_binary(
    name = "thing",
    srcs = ["thing.cc"],
    data = ["BUILD"],
)
EOF
  cat > thing.cc <<EOF
int main() { return 0; }
EOF
  bazel build //:thing $EXTRA_BUILD_FLAGS &> $TEST_log || fail "Build failed"
  [[ -d ${PRODUCT_NAME}-bin/thing${EXT}.runfiles/foo ]] || fail "foo not found"

  # Change workspace name to bar.
  sed -ie 's,workspace(.*,workspace(name = "bar"),' WORKSPACE
  bazel build //:thing $EXTRA_BUILD_FLAGS &> $TEST_log || fail "Build failed"
  [[ -d ${PRODUCT_NAME}-bin/thing${EXT}.runfiles/bar ]] || fail "bar not found"
  [[ ! -d ${PRODUCT_NAME}-bin/thing${EXT}.runfiles/foo ]] \
    || fail "Old foo still found"
}

# regression test for b/237547165
function test_fail_on_middleman_in_transitive_runfiles_for_executable() {
  cat > rule.bzl <<EOF
def _impl(ctx):
    exe = ctx.actions.declare_file(ctx.label.name + '.out')
    ctx.actions.write(exe, "")
    internal_outputs = ctx.attr.bin[OutputGroupInfo]._hidden_top_level_INTERNAL_
    runfiles = ctx.runfiles(transitive_files = internal_outputs)
    return DefaultInfo(runfiles = runfiles, executable = exe)
bad_runfiles = rule(
  implementation = _impl,
  attrs = {"bin" : attr.label()},
  executable = True,
)
EOF
  cat > BUILD <<EOF
load(":rule.bzl", "bad_runfiles");
cc_binary(
    name = "thing",
    srcs = ["thing.cc"],
)
bad_runfiles(name = "test", bin = ":thing")
EOF
  cat > thing.cc <<EOF
int main() { return 0; }
EOF
  bazel build //:test &> $TEST_log && fail "Expected build to fail but it succeeded"
  expect_log_once "Runfiles must not contain middleman artifacts"
}

function test_manifest_action_reruns_on_output_base_change() {
  CURRENT_DIRECTORY=$(pwd)
  if $is_windows; then
    CURRENT_DIRECTORY=$(cygpath -m "${CURRENT_DIRECTORY}")
  fi

  if $is_windows; then
    MANIFEST_PATH=bazel-bin/hello_world.exe.runfiles_manifest
  else
    MANIFEST_PATH=bazel-bin/hello_world.runfiles_manifest
  fi

  OUTPUT_BASE="${CURRENT_DIRECTORY}/test/outputs/__main__"
  TEST_FOLDER_1="${CURRENT_DIRECTORY}/test/test1/$(basename ${CURRENT_DIRECTORY})"
  TEST_FOLDER_2="${CURRENT_DIRECTORY}/test/test2/$(basename ${CURRENT_DIRECTORY})"

  mkdir -p "${OUTPUT_BASE}"
  mkdir -p "${TEST_FOLDER_1}"
  mkdir -p "${TEST_FOLDER_2}"

  cat > BUILD <<EOF
sh_binary(
    name = "hello_world",
    srcs = ["hello_world.sh"],
)
EOF
  cat > hello_world.sh <<EOF
echo "Hello World"
EOF
  chmod +x hello_world.sh

  for d in $(ls -a | grep -v '^test$' | grep -v '^\.*$'); do
    cp -R "${CURRENT_DIRECTORY}/${d}" "${TEST_FOLDER_1}"
    cp -R "${CURRENT_DIRECTORY}/${d}" "${TEST_FOLDER_2}"
  done

  cd "${TEST_FOLDER_1}"
  bazel --output_base="${OUTPUT_BASE}" build //:hello_world
  assert_contains "${TEST_FOLDER_1}" "${MANIFEST_PATH}"
  assert_not_contains "${TEST_FOLDER_2}" "${MANIFEST_PATH}"

  cd "${TEST_FOLDER_2}"
  bazel --output_base="${OUTPUT_BASE}" build //:hello_world
  assert_not_contains "${TEST_FOLDER_1}" "${MANIFEST_PATH}"
  assert_contains "${TEST_FOLDER_2}" "${MANIFEST_PATH}"
}

function test_removal_of_old_tempfiles() {
  cat > BUILD << EOF
sh_binary(
    name = "foo",
    srcs = ["foo.sh"],
)
EOF
  touch foo.sh
  chmod +x foo.sh

  # Build once to create a runfiles directory.
  bazel build //:foo $EXTRA_BUILD_FLAGS >&$TEST_log || fail "build failed"

  # Remove the MANIFEST file that was created by the previous build.
  # Create an inaccessible file in the place where build-runfiles writes
  # its temporary results.
  #
  # This simulates the case where the runfiles creation process is
  # interrupted and leaves the temporary file behind. The temporary file
  # may become read-only if it was stored in a snapshot.
  rm ${PRODUCT_NAME}-bin/foo${EXT}.runfiles/MANIFEST
  touch ${PRODUCT_NAME}-bin/foo${EXT}.runfiles/MANIFEST.tmp
  chmod 0 ${PRODUCT_NAME}-bin/foo${EXT}.runfiles/MANIFEST.tmp

  # Even with the inaccessible temporary file in place, build-runfiles
  # should complete successfully. The MANIFEST file should be recreated.
  bazel build //:foo $EXTRA_BUILD_FLAGS >&$TEST_log || fail "build failed"
  [[ -f ${PRODUCT_NAME}-bin/foo${EXT}.runfiles/MANIFEST ]] \
    || fail "MANIFEST file not recreated"
}

function test_rebuilt_when_mapping_changes {
  if "$is_windows"; then
    # Can't do 'ln -s' on Windows
    return
  fi

  mkdir -p a
  cat > a/a.bzl <<'EOF'
def _a_impl(ctx):
    ex = ctx.actions.declare_file("a.sh")
    r = ctx.runfiles(
        files = [ex],
        symlinks = {ctx.attr.link: ctx.file.target},
    )
    ctx.actions.write(ex, "#!/bin/bash", True)
    return DefaultInfo(
        files = depset([ex]),
        default_runfiles = r,
        executable = ex,
    )

a = rule(
    implementation = _a_impl,
    executable = True,
    attrs = {
        "link": attr.string(),
        "target": attr.label(allow_single_file = True),
    },
)
EOF

  cat >a/BUILD <<'EOF'
load(":a.bzl", "a")
a(
  name = "a",
  link = "link_one",
  target = ":f",
)

genrule(
  name = "g",
  srcs = [],
  outs = ["go"],
  output_to_bindir = 1,
  tools = [":a"],
  cmd = "echo $(location :a).runfiles/*/link_* > $@",
)
EOF

  touch a/f

  bazel build //a:g || fail "first build failed"
  assert_contains '/link_one$' *-bin/a/go

  inplace-sed 's/link_one/link_two/' a/BUILD
  bazel build //a:g || fail "first build failed"
  assert_contains '/link_two$' *-bin/a/go
}

run_suite "runfiles"