Allow overriding all external repositories used by our integration tests.
Downloading and extracting external repositories over and over again is the biggest performance and SSD killer in our integration tests.
This PR introduces a mechanism where the integration tests can reuse the repositories downloaded by the "outer" Bazel. It also means we can disable network access for most tests, increasing hermeticity.
RELNOTES: None.
PiperOrigin-RevId: 291901125
diff --git a/src/test/java/com/google/devtools/build/lib/blackbox/bazel/DefaultToolsSetup.java b/src/test/java/com/google/devtools/build/lib/blackbox/bazel/DefaultToolsSetup.java
index 00806fc..c549ef6 100644
--- a/src/test/java/com/google/devtools/build/lib/blackbox/bazel/DefaultToolsSetup.java
+++ b/src/test/java/com/google/devtools/build/lib/blackbox/bazel/DefaultToolsSetup.java
@@ -14,20 +14,63 @@
package com.google.devtools.build.lib.blackbox.bazel;
+import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.blackbox.framework.BlackBoxTestContext;
import com.google.devtools.build.lib.blackbox.framework.ToolsSetup;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
/** Setup for Bazel default tools */
public class DefaultToolsSetup implements ToolsSetup {
+
+ private static ImmutableList<String> repos =
+ ImmutableList.<String>builder().add("rules_cc").add("rules_proto").add("rules_java").build();
+
+ private ImmutableList<String> getRepositoryOverrides() {
+ String sharedRepoHome = System.getenv("TEST_REPOSITORY_HOME");
+ if (sharedRepoHome == null) {
+ return ImmutableList.of();
+ }
+
+ Path sharedRepoHomePath = Paths.get(sharedRepoHome);
+ if (!Files.exists(sharedRepoHomePath)) {
+ return ImmutableList.of();
+ }
+
+ ImmutableList.Builder<String> lines = ImmutableList.builder();
+ for (String repo : repos) {
+ Path sharedRepoPath = sharedRepoHomePath.resolve(repo);
+ lines.add(
+ "common --override_repository="
+ + repo
+ + "="
+ + sharedRepoPath.toString().replace('\\', '/'));
+ }
+
+ return lines.build();
+ }
+
@Override
public void setup(BlackBoxTestContext context) throws IOException {
Path outputRoot = Files.createTempDirectory(context.getTmpDir(), "root").toAbsolutePath();
- context.write(
- ".bazelrc",
- "startup --output_user_root=" + outputRoot.toString().replace('\\', '/'),
- "build -j 8");
+ ArrayList<String> lines = new ArrayList<>();
+ lines.add("startup --output_user_root=" + outputRoot.toString().replace('\\', '/'));
+ lines.addAll(getRepositoryOverrides());
+
+ String sharedInstallBase = System.getenv("TEST_INSTALL_BASE");
+ if (sharedInstallBase != null) {
+ lines.add("startup --install_base=" + sharedInstallBase);
+ }
+
+ String sharedRepoCache = System.getenv("REPOSITORY_CACHE");
+ if (sharedRepoCache != null) {
+ lines.add("common --repository_cache=" + sharedRepoCache);
+ lines.add("common --experimental_repository_cache_hardlinks");
+ }
+
+ context.write(".bazelrc", lines);
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/blackbox/framework/BlackBoxTestContext.java b/src/test/java/com/google/devtools/build/lib/blackbox/framework/BlackBoxTestContext.java
index 5ccd9f9..f9a0328 100644
--- a/src/test/java/com/google/devtools/build/lib/blackbox/framework/BlackBoxTestContext.java
+++ b/src/test/java/com/google/devtools/build/lib/blackbox/framework/BlackBoxTestContext.java
@@ -103,6 +103,21 @@
}
/**
+ * Writes <code>lines</code> using ISO_8859_1 into the file, specified by the <code>subPath</code>
+ * relative to the working directory. Overrides the file if it exists, creates the file if it does
+ * not exist.
+ *
+ * @param subPath path to file relative to working directory
+ * @param lines lines of text to write. Newlines are added by the method.
+ * @return Path to the file
+ * @throws IOException in case if the file can not be created/overridden, or can not be open for
+ * writing
+ */
+ public Path write(String subPath, List<String> lines) throws IOException {
+ return PathUtils.writeFileInDir(workDir, subPath, lines);
+ }
+
+ /**
* Writes <code>lines</code> using ISO_8859_1 into the BUILD file in the directory, specified by
* the <code>subPath</code> relative to the working directory. Overrides the file if it exists,
* creates the file if it does not exist.
diff --git a/src/test/java/com/google/devtools/build/lib/blackbox/framework/PathUtils.java b/src/test/java/com/google/devtools/build/lib/blackbox/framework/PathUtils.java
index 3047352..9052659 100644
--- a/src/test/java/com/google/devtools/build/lib/blackbox/framework/PathUtils.java
+++ b/src/test/java/com/google/devtools/build/lib/blackbox/framework/PathUtils.java
@@ -245,6 +245,21 @@
}
/**
+ * Writes the file in the <code>directory/subPath</code> location using ISO_8859_1. Overrides the
+ * file if it exists, creates the file if it does not exist.
+ *
+ * @param directory root directory, under which the subtree with the file is created
+ * @param subPath path under <code>directory</code>, under which the file is created
+ * @param lines lines to be written
+ * @return Path to created file
+ * @throws IOException in case file can not be written
+ */
+ public static Path writeFileInDir(Path directory, String subPath, List<String> lines)
+ throws IOException {
+ return writeFile(resolve(directory, subPath), lines);
+ }
+
+ /**
* Writes the file in the <code>path</code> location using ISO_8859_1. Overrides the file if it
* exists, creates the file if it does not exist.
*
@@ -258,6 +273,19 @@
}
/**
+ * Writes the file in the <code>path</code> location using ISO_8859_1. Overrides the file if it
+ * exists, creates the file if it does not exist.
+ *
+ * @param path location where to write the file
+ * @param lines lines to be written
+ * @throws IOException in case file can not be written
+ */
+ public static Path writeFile(Path path, List<String> lines) throws IOException {
+ Files.createDirectories(path.getParent());
+ return Files.write(path, lines, StandardCharsets.ISO_8859_1);
+ }
+
+ /**
* Writes the BUILD file under <code>directory</code> using ISO_8859_1. Overrides the file if it
* exists, creates the file if it does not exist.
*
diff --git a/src/test/py/bazel/test_base.py b/src/test/py/bazel/test_base.py
index fc77b3d..b32fe7f 100644
--- a/src/test/py/bazel/test_base.py
+++ b/src/test/py/bazel/test_base.py
@@ -52,6 +52,20 @@
_worker_proc = None
_cas_path = None
+ _SHARED_REPOS = (
+ 'rules_cc',
+ 'rules_java',
+ 'rules_proto',
+ 'remotejdk11_linux_for_testing',
+ 'remotejdk11_linux_aarch64_for_testing',
+ 'remotejdk11_macos_for_testing',
+ 'remotejdk11_win_for_testing',
+ 'remote_java_tools_darwin_for_testing',
+ 'remote_java_tools_linux_for_testing',
+ 'remote_java_tools_windows_for_testing',
+ 'remote_coverage_tools_for_testing',
+ )
+
def setUp(self):
unittest.TestCase.setUp(self)
if self._runfiles is None:
@@ -63,7 +77,19 @@
self._test_cwd = tempfile.mkdtemp(dir=self._tests_root)
self._test_bazelrc = os.path.join(self._temp, 'test_bazelrc')
with open(self._test_bazelrc, 'wt') as f:
- f.write('build --jobs=8\n')
+ shared_repo_home = os.environ.get('TEST_REPOSITORY_HOME')
+ if shared_repo_home and os.path.exists(shared_repo_home):
+ for repo in self._SHARED_REPOS:
+ f.write('common --override_repository={}={}\n'.format(
+ repo.replace('_for_testing', ''),
+ os.path.join(shared_repo_home, repo).replace('\\', '/')))
+ shared_install_base = os.environ.get('TEST_INSTALL_BASE')
+ if shared_install_base:
+ f.write('startup --install_base={}\n'.format(shared_install_base))
+ shared_repo_cache = os.environ.get('REPOSITORY_CACHE')
+ if shared_repo_cache:
+ f.write('common --repository_cache={}\n'.format(shared_repo_cache))
+ f.write('common --experimental_repository_cache_hardlinks\n')
os.chdir(self._test_cwd)
def tearDown(self):
diff --git a/src/test/shell/testenv.sh b/src/test/shell/testenv.sh
index 07d50f7..7570363 100755
--- a/src/test/shell/testenv.sh
+++ b/src/test/shell/testenv.sh
@@ -282,19 +282,79 @@
${EXTRA_BAZELRC:-}
EOF
+ if [[ -n ${TEST_REPOSITORY_HOME:-} ]]; then
+ echo "testenv.sh: Using shared repositories from $TEST_REPOSITORY_HOME."
+
+ repos=(
+ "android_tools_for_testing"
+ "bazel_toolchains"
+ "com_google_protobuf"
+ "openjdk10_darwin_archive"
+ "openjdk10_linux_archive"
+ "openjdk10_windows_archive"
+ "openjdk11_darwin_archive"
+ "openjdk11_linux_archive"
+ "openjdk11_windows_archive"
+ "openjdk12_darwin_archive"
+ "openjdk12_linux_archive"
+ "openjdk12_windows_archive"
+ "openjdk9_darwin_archive"
+ "openjdk9_linux_archive"
+ "openjdk9_windows_archive"
+ "openjdk_linux_aarch64_minimal"
+ "openjdk_linux_minimal"
+ "openjdk_macos_minimal"
+ "openjdk_win_minimal"
+ "remote_coverage_tools_for_testing"
+ "remote_java_tools_darwin_for_testing"
+ "remote_java_tools_javac10_test_darwin"
+ "remote_java_tools_javac10_test_linux"
+ "remote_java_tools_javac10_test_windows"
+ "remote_java_tools_javac11_test_darwin"
+ "remote_java_tools_javac11_test_linux"
+ "remote_java_tools_javac11_test_windows"
+ "remote_java_tools_javac12_test_darwin"
+ "remote_java_tools_javac12_test_linux"
+ "remote_java_tools_javac12_test_windows"
+ "remote_java_tools_javac9_test_darwin"
+ "remote_java_tools_javac9_test_linux"
+ "remote_java_tools_javac9_test_windows"
+ "remote_java_tools_linux_for_testing"
+ "remote_java_tools_windows_for_testing"
+ "remotejdk10_linux_for_testing"
+ "remotejdk10_linux_aarch64_for_testing"
+ "remotejdk10_macos_for_testing"
+ "remotejdk10_win_for_testing"
+ "remotejdk11_linux_for_testing"
+ "remotejdk11_linux_aarch64_for_testing"
+ "remotejdk11_macos_for_testing"
+ "remotejdk11_win_for_testing"
+ "remotejdk_linux_for_testing"
+ "remotejdk_linux_aarch64_for_testing"
+ "remotejdk_macos_for_testing"
+ "remotejdk_win_for_testing"
+ "rules_cc"
+ "rules_java"
+ "rules_pkg"
+ "rules_proto"
+ "rules_python"
+ )
+ for repo in "${repos[@]}"; do
+ reponame="${repo%"_for_testing"}"
+ echo "common --override_repository=$reponame=$TEST_REPOSITORY_HOME/$repo" >> $TEST_TMPDIR/bazelrc
+ done
+ fi
+
if [[ -n ${REPOSITORY_CACHE:-} ]]; then
echo "testenv.sh: Using repository cache at $REPOSITORY_CACHE."
cat >>$TEST_TMPDIR/bazelrc <<EOF
-sync --repository_cache=$REPOSITORY_CACHE --experimental_repository_cache_hardlinks
-fetch --repository_cache=$REPOSITORY_CACHE --experimental_repository_cache_hardlinks
-build --repository_cache=$REPOSITORY_CACHE --experimental_repository_cache_hardlinks
-query --repository_cache=$REPOSITORY_CACHE --experimental_repository_cache_hardlinks
+common --repository_cache=$REPOSITORY_CACHE --experimental_repository_cache_hardlinks
EOF
fi
- if [[ -n ${INSTALL_BASE:-} ]]; then
- echo "testenv.sh: Using shared install base at $INSTALL_BASE."
- echo "startup --install_base=$INSTALL_BASE" >> $TEST_TMPDIR/bazelrc
+ if [[ -n ${TEST_INSTALL_BASE:-} ]]; then
+ echo "testenv.sh: Using shared install base at $TEST_INSTALL_BASE."
+ echo "startup --install_base=$TEST_INSTALL_BASE" >> $TEST_TMPDIR/bazelrc
fi
}
