Enable git_submodules tests and merge git_repository and starlark_git_repository
- Added necessary flags to allow clone submodule from local repository.
- We only have the starlark version of git repository rule and most test cases in two integration tests are duplicated.
Fixes https://github.com/bazelbuild/bazel/issues/17040
RELNOTES: None
PiperOrigin-RevId: 498989149
Change-Id: I1eec3efa17382ddd0a608c0984e2f6c9f29c5c7c
diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
index 21fbf38..c96c33d 100644
--- a/src/test/shell/bazel/BUILD
+++ b/src/test/shell/bazel/BUILD
@@ -717,17 +717,6 @@
)
sh_test(
- name = "git_repository_test",
- size = "large",
- srcs = ["git_repository_test.sh"],
- data = [
- ":test-deps",
- "//src/test/shell/bazel/testdata:git-repos",
- "@bazel_tools//tools/bash/runfiles",
- ],
-)
-
-sh_test(
name = "starlark_git_repository_test",
size = "large",
srcs = ["starlark_git_repository_test.sh"],
diff --git a/src/test/shell/bazel/git_repository_test.sh b/src/test/shell/bazel/git_repository_test.sh
deleted file mode 100755
index 05ff5ca..0000000
--- a/src/test/shell/bazel/git_repository_test.sh
+++ /dev/null
@@ -1,519 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2015 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Test git_repository and new_git_repository workspace rules.
-#
-
-set -euo pipefail
-# --- begin runfiles.bash initialization ---
-if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
- if [[ -f "$0.runfiles_manifest" ]]; then
- export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
- elif [[ -f "$0.runfiles/MANIFEST" ]]; then
- export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
- elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
- export RUNFILES_DIR="$0.runfiles"
- fi
-fi
-if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
- source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
-elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
- source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
- "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
-else
- echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
- exit 1
-fi
-# --- end runfiles.bash initialization ---
-
-source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \
- || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
-
-# `uname` returns the current platform, e.g "MSYS_NT-10.0" or "Linux".
-# `tr` converts all upper case letters to lower case.
-# `case` matches the result if the `uname | tr` expression to string prefixes
-# that use the same wildcards as names do in Bash, i.e. "msys*" matches strings
-# starting with "msys", and "*" matches everything (it's the default case).
-case "$(uname -s | tr [:upper:] [:lower:])" in
-msys*)
- # As of 2019-01-15, Bazel on Windows only supports MSYS Bash.
- declare -r is_windows=true
- ;;
-*)
- declare -r is_windows=false
- ;;
-esac
-
-if $is_windows; then
- export MSYS_NO_PATHCONV=1
- export MSYS2_ARG_CONV_EXCL="*"
- # Enable symlink runfiles tree to make bazel run work
- add_to_bazelrc "build --enable_runfiles"
-fi
-
-# Global test setup.
-#
-# Unpacks the test Git repositories in the test temporary directory.
-function set_up() {
- bazel clean --expunge
- local repos_dir=$TEST_TMPDIR/repos
- if [ -e "$repos_dir" ]; then
- rm -rf $repos_dir
- fi
-
- mkdir -p $repos_dir
- cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/pluto-repo.tar.gz)" $repos_dir
- cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/outer-planets-repo.tar.gz)" $repos_dir
- cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/refetch-repo.tar.gz)" $repos_dir
- cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/strip-prefix-repo.tar.gz)" $repos_dir
- cd $repos_dir
- tar zxf pluto-repo.tar.gz
- tar zxf outer-planets-repo.tar.gz
- tar zxf refetch-repo.tar.gz
- tar zxf strip-prefix-repo.tar.gz
-}
-
-# Test cloning a Git repository using the git_repository rule.
-#
-# This test uses the pluto Git repository at tag 1-build, which contains the
-# following files:
-#
-# pluto/
-# WORKSPACE
-# BUILD
-# info
-#
-# Then, set up workspace with the following files:
-#
-# $WORKSPACE_DIR/
-# WORKSPACE
-# planets/
-# BUILD
-# planet_info.sh
-#
-# //planets has a dependency on a target in the pluto Git repository.
-function test_git_repository() {
- local pluto_repo_dir=$TEST_TMPDIR/repos/pluto
- # Commit corresponds to tag 1-build. See testdata/pluto.git_log.
- local commit_hash="52f9a3f87a2dd17ae0e5847bbae9734f09354afd"
-
- # Create a workspace that clones the repository at the first commit.
- cd $WORKSPACE_DIR
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-git_repository(
- name = "pluto",
- remote = "$pluto_repo_dir",
- commit = "$commit_hash",
-)
-EOF
- mkdir -p planets
- cat > planets/BUILD <<EOF
-sh_binary(
- name = "planet-info",
- srcs = ["planet_info.sh"],
- data = ["@pluto//:pluto"],
-)
-EOF
-
- cat > planets/planet_info.sh <<EOF
-#!/bin/sh
-cat ../pluto/info
-EOF
- chmod +x planets/planet_info.sh
-
- bazel run //planets:planet-info >& $TEST_log \
- || echo "Expected build/run to succeed"
- expect_log "Pluto is a dwarf planet"
-}
-
-function test_new_git_repository_with_build_file() {
- do_new_git_repository_test "build_file"
-}
-
-function test_new_git_repository_with_build_file_content() {
- do_new_git_repository_test "build_file_content"
-}
-
-# Test cloning a Git repository using the new_git_repository rule.
-#
-# This test uses the pluto Git repository at tag 0-initial, which contains the
-# following files:
-#
-# pluto/
-# info
-#
-# Set up workspace with the following files:
-#
-# $WORKSPACE_DIR/
-# WORKSPACE
-# pluto.BUILD
-# planets/
-# BUILD
-# planet_info.sh
-#
-# //planets has a dependency on a target in the $TEST_TMPDIR/pluto Git
-# repository.
-function do_new_git_repository_test() {
- local pluto_repo_dir=$TEST_TMPDIR/repos/pluto
-
- # Create a workspace that clones the repository at the first commit.
- cd $WORKSPACE_DIR
-
- if [ "$1" == "build_file" ] ; then
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
-new_git_repository(
- name = "pluto",
- remote = "$pluto_repo_dir",
- tag = "0-initial",
- build_file = "//:pluto.BUILD",
-)
-EOF
- cat > BUILD <<EOF
-exports_files(['pluto.BUILD'])
-EOF
- cat > pluto.BUILD <<EOF
-filegroup(
- name = "pluto",
- srcs = ["info"],
- visibility = ["//visibility:public"],
-)
-EOF
- else
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
-new_git_repository(
- name = "pluto",
- remote = "$pluto_repo_dir",
- tag = "0-initial",
- build_file_content = """
-filegroup(
- name = "pluto",
- srcs = ["info"],
- visibility = ["//visibility:public"],
-)"""
-)
-EOF
- fi
-
- mkdir -p planets
- cat > planets/BUILD <<EOF
-sh_binary(
- name = "planet-info",
- srcs = ["planet_info.sh"],
- data = ["@pluto//:pluto"],
-)
-EOF
-
- cat > planets/planet_info.sh <<EOF
-#!/bin/sh
-cat ../pluto/info
-EOF
- chmod +x planets/planet_info.sh
-
- bazel run //planets:planet-info >& $TEST_log \
- || echo "Expected build/run to succeed"
- expect_log "Pluto is a planet"
-}
-
-# Test cloning a Git repository that has a submodule using the
-# new_git_repository rule.
-#
-# This test uses the outer-planets Git repository at revision 1-submodule, which
-# contains the following files:
-#
-# outer_planets/
-# neptune/
-# info
-# pluto/ --> submodule ../pluto
-# info
-#
-# Set up workspace with the following files:
-#
-# $WORKSPACE_DIR/
-# WORKSPACE
-# outer_planets.BUILD
-# planets/
-# BUILD
-# planet_info.sh
-#
-# planets has a dependency on targets in the $TEST_TMPDIR/outer_planets Git
-# repository.
-function test_new_git_repository_submodules() {
- # Temporarily disable on Windows due to https://github.com/bazelbuild/bazel/issues/17040
- if $is_windows; then
- return
- fi
- local outer_planets_repo_dir=$TEST_TMPDIR/repos/outer-planets
-
- # Create a workspace that clones the outer_planets repository.
- cd $WORKSPACE_DIR
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
-new_git_repository(
- name = "outer_planets",
- remote = "$outer_planets_repo_dir",
- tag = "1-submodule",
- init_submodules = 1,
- build_file = "//:outer_planets.BUILD",
-)
-EOF
-
- cat > BUILD <<EOF
-exports_files(['outer_planets.BUILD'])
-EOF
- cat > outer_planets.BUILD <<EOF
-filegroup(
- name = "neptune",
- srcs = ["neptune/info"],
- visibility = ["//visibility:public"],
-)
-
-filegroup(
- name = "pluto",
- srcs = ["pluto/info"],
- visibility = ["//visibility:public"],
-)
-EOF
-
- mkdir -p planets
- cat > planets/BUILD <<EOF
-sh_binary(
- name = "planet-info",
- srcs = ["planet_info.sh"],
- data = [
- "@outer_planets//:neptune",
- "@outer_planets//:pluto",
- ],
-)
-EOF
-
- cat > planets/planet_info.sh <<EOF
-#!/bin/sh
-cat ../outer_planets/neptune/info
-cat ../outer_planets/pluto/info
-EOF
- chmod +x planets/planet_info.sh
-
- bazel run //planets:planet-info >& $TEST_log \
- || echo "Expected build/run to succeed"
- expect_log "Neptune is a planet"
- expect_log "Pluto is a planet"
-}
-
-function test_git_repository_not_refetched_on_server_restart() {
- local repo_dir=$TEST_TMPDIR/repos/refetch
-
- cd $WORKSPACE_DIR
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-git_repository(name='g', remote='$repo_dir', commit='22095302abaf776886879efa5129aa4d44c53017')
-EOF
-
- # Use batch to force server restarts.
- bazel --batch build @g//:g >& $TEST_log || fail "Build failed"
- assert_contains "GIT 1" bazel-genfiles/external/g/go
-
- # Without changing anything, restart the server, which should not cause the checkout to be re-cloned.
- bazel --batch build @g//:g >& $TEST_log || fail "Build failed"
- assert_contains "GIT 1" bazel-genfiles/external/g/go
-
- # Change the commit id, which should cause the checkout to be re-cloned.
- rm WORKSPACE
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-git_repository(name='g', remote='$repo_dir', commit='db134ae9b644d8237954a8e6f1ef80fcfd85d521')
-EOF
-
- bazel --batch build @g//:g >& $TEST_log || fail "Build failed"
- assert_contains "GIT 2" bazel-genfiles/external/g/go
-
- # Change the WORKSPACE but not the commit id, which should not cause the checkout to be re-cloned.
- rm WORKSPACE
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-# This comment line is to change the line numbers, which should not cause Bazel
-# to refetch the repository
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-git_repository(name='g', remote='$repo_dir', commit='db134ae9b644d8237954a8e6f1ef80fcfd85d521')
-EOF
-
- bazel --batch build @g//:g >& $TEST_log || fail "Build failed"
- assert_contains "GIT 2" bazel-genfiles/external/g/go
-}
-
-function test_git_repository_refetched_when_commit_changes() {
- local repo_dir=$TEST_TMPDIR/repos/refetch
-
- cd $WORKSPACE_DIR
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-git_repository(name='g', remote='$repo_dir', commit='22095302abaf776886879efa5129aa4d44c53017')
-EOF
-
- bazel build @g//:g >& $TEST_log || fail "Build failed"
- assert_contains "GIT 1" bazel-genfiles/external/g/go
-
- # Change the commit id, which should cause the checkout to be re-cloned.
- rm WORKSPACE
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-git_repository(name='g', remote='$repo_dir', commit='db134ae9b644d8237954a8e6f1ef80fcfd85d521')
-EOF
-
- bazel build @g//:g >& $TEST_log || fail "Build failed"
- assert_contains "GIT 2" bazel-genfiles/external/g/go
-}
-
-function test_git_repository_and_nofetch() {
- local repo_dir=$TEST_TMPDIR/repos/refetch
-
- cd $WORKSPACE_DIR
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-git_repository(name='g', remote='$repo_dir', commit='22095302abaf776886879efa5129aa4d44c53017')
-EOF
-
- bazel build --nofetch @g//:g >& $TEST_log && fail "Build succeeded"
- expect_log "fetching repositories is disabled"
- bazel build @g//:g >& $TEST_log || fail "Build failed"
- assert_contains "GIT 1" bazel-genfiles/external/g/go
-
- rm WORKSPACE
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-git_repository(name='g', remote='$repo_dir', commit='db134ae9b644d8237954a8e6f1ef80fcfd85d521')
-EOF
-
- bazel build --nofetch @g//:g >& $TEST_log || fail "Build failed"
- expect_log "External repository 'g' is not up-to-date"
- assert_contains "GIT 1" bazel-genfiles/external/g/go
- bazel build @g//:g >& $TEST_log || fail "Build failed"
- assert_contains "GIT 2" bazel-genfiles/external/g/go
-}
-
-# Helper function for setting up the workspace as follows
-#
-# $WORKSPACE_DIR/
-# WORKSPACE
-# planets/
-# planet_info.sh
-# BUILD
-function setup_error_test() {
- cd $WORKSPACE_DIR
- mkdir -p planets
- cat > planets/planet_info.sh <<EOF
-#!/bin/sh
-cat external/pluto/info
-EOF
-
- cat > planets/BUILD <<EOF
-sh_binary(
- name = "planet-info",
- srcs = ["planet_info.sh"],
- data = ["@pluto//:pluto"],
-)
-EOF
-}
-
-# Verifies that rule fails if both tag and commit are set.
-#
-# This test uses the pluto Git repository at tag 1-build, which contains the
-# following files:
-#
-# pluto/
-# WORKSPACE
-# BUILD
-# info
-function test_git_repository_both_commit_tag_error() {
- setup_error_test
- local pluto_repo_dir=$TEST_TMPDIR/pluto
- # Commit corresponds to tag 1-build. See testdata/pluto.git_log.
- local commit_hash="52f9a3f87a2dd17ae0e5847bbae9734f09354afd"
-
- cd $WORKSPACE_DIR
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-git_repository(
- name = "pluto",
- remote = "$pluto_repo_dir",
- tag = "1-build",
- commit = "$commit_hash",
-)
-EOF
-
- bazel fetch //planets:planet-info >& $TEST_log \
- || echo "Expect run to fail."
- expect_log "Exactly one of commit"
-}
-
-# Verifies that rule fails if neither tag or commit are set.
-#
-# This test uses the pluto Git repository at tag 1-build, which contains the
-# following files:
-#
-# pluto/
-# WORKSPACE
-# BUILD
-# info
-function test_git_repository_no_commit_tag_error() {
- setup_error_test
- local pluto_repo_dir=$TEST_TMPDIR/pluto
-
- cd $WORKSPACE_DIR
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-git_repository(
- name = "pluto",
- remote = "$pluto_repo_dir",
-)
-EOF
-
- bazel fetch //planets:planet-info >& $TEST_log \
- || echo "Expect run to fail."
- expect_log "Exactly one of commit"
-}
-
-# Verifies that load statement works while using strip_prefix.
-#
-# This test uses the strip-prefix Git repository, which contains the
-# following files:
-#
-# strip-prefix
-# └── prefix-foo
-# ├── BUILD
-# ├── WORKSPACE
-# └── defs.bzl
-function test_git_repository_with_strip_prefix_for_load_statement() {
- setup_error_test
- local strip_prefix_repo_dir=$TEST_TMPDIR/repos/strip-prefix
-
- cd $WORKSPACE_DIR
- cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-git_repository(
- name = "foo",
- remote = "$strip_prefix_repo_dir",
- commit = "f8167a60de4460e89601724fb13b4fc505da3f3d",
- strip_prefix = "prefix-foo",
-)
-load("@foo//:defs.bzl", "FOO")
-EOF
-
- bazel info >& $TEST_log || fail "Expect bazel info to succeed."
-}
-
-run_suite "git_repository tests"
diff --git a/src/test/shell/bazel/starlark_git_repository_test.sh b/src/test/shell/bazel/starlark_git_repository_test.sh
index 42f00b1..2e1e9ca 100755
--- a/src/test/shell/bazel/starlark_git_repository_test.sh
+++ b/src/test/shell/bazel/starlark_git_repository_test.sh
@@ -78,10 +78,12 @@
cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/pluto-repo.tar.gz)" $repos_dir
cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/outer-planets-repo.tar.gz)" $repos_dir
cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/refetch-repo.tar.gz)" $repos_dir
+ cp "$(rlocation io_bazel/src/test/shell/bazel/testdata/strip-prefix-repo.tar.gz)" $repos_dir
cd $repos_dir
tar zxf pluto-repo.tar.gz
tar zxf outer-planets-repo.tar.gz
tar zxf refetch-repo.tar.gz
+ tar zxf strip-prefix-repo.tar.gz
# Fix environment variables for a hermetic use of git.
export GIT_CONFIG_NOSYSTEM=1
@@ -314,10 +316,6 @@
# planets has a dependency on targets in the $TEST_TMPDIR/outer_planets Git
# repository.
function test_new_git_repository_submodules() {
- # Temporarily disable on Windows due to https://github.com/bazelbuild/bazel/issues/17040
- if $is_windows; then
- return
- fi
local outer_planets_repo_dir=$TEST_TMPDIR/repos/outer-planets
# Create a workspace that clones the outer_planets repository.
@@ -375,11 +373,7 @@
expect_log "Pluto is a planet"
}
-function test_new_git_repository_submodules() {
- # Temporarily disable on Windows due to https://github.com/bazelbuild/bazel/issues/17040
- if $is_windows; then
- return
- fi
+function test_new_git_repository_submodules_with_recursive_init_modules() {
local outer_planets_repo_dir=$TEST_TMPDIR/repos/outer-planets
# Create a workspace that clones the outer_planets repository.
@@ -694,4 +688,33 @@
expect_log "shallow_since not allowed if a tag is specified; --depth=1 will be used for tags"
}
+# Verifies that load statement works while using strip_prefix.
+#
+# This test uses the strip-prefix Git repository, which contains the
+# following files:
+#
+# strip-prefix
+# └── prefix-foo
+# ├── BUILD
+# ├── WORKSPACE
+# └── defs.bzl
+function test_git_repository_with_strip_prefix_for_load_statement() {
+ setup_error_test
+ local strip_prefix_repo_dir=$TEST_TMPDIR/repos/strip-prefix
+
+ cd $WORKSPACE_DIR
+ cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+git_repository(
+ name = "foo",
+ remote = "$strip_prefix_repo_dir",
+ commit = "f8167a60de4460e89601724fb13b4fc505da3f3d",
+ strip_prefix = "prefix-foo",
+)
+load("@foo//:defs.bzl", "FOO")
+EOF
+
+ bazel info >& $TEST_log || fail "Expect bazel info to succeed."
+}
+
run_suite "Starlark git_repository tests"
diff --git a/tools/build_defs/repo/git_worker.bzl b/tools/build_defs/repo/git_worker.bzl
index 4983604..c663cda 100644
--- a/tools/build_defs/repo/git_worker.bzl
+++ b/tools/build_defs/repo/git_worker.bzl
@@ -152,9 +152,12 @@
def update_submodules(ctx, git_repo, recursive = False):
if recursive:
- _git(ctx, git_repo, "submodule", "update", "--init", "--recursive", "--checkout", "--force")
+ # "protocol.file.allow=always" allows the submodule command clone from a local directory.
+ # It's necessary for Git 2.38.1 and assoicated backport versions.
+ # See https://github.com/bazelbuild/bazel/issues/17040
+ _git(ctx, git_repo, "-c", "protocol.file.allow=always", "submodule", "update", "--init", "--recursive", "--checkout", "--force")
else:
- _git(ctx, git_repo, "submodule", "update", "--init", "--checkout", "--force")
+ _git(ctx, git_repo, "-c", "protocol.file.allow=always", "submodule", "update", "--init", "--checkout", "--force")
def _get_head_commit(ctx, git_repo):
return _git(ctx, git_repo, "log", "-n", "1", "--pretty=format:%H")
