For Skymeld, eagerly plant the symlinks from execroot to the source root.
This enables Skymeld to run local actions for builds with a single package_path.
PiperOrigin-RevId: 440334907
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
index 7790816..c5aaf15 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
@@ -205,7 +205,11 @@
Set<AspectKey> builtAspects = new HashSet<>();
try (SilentCloseable c = Profiler.instance().profile("ExecutionTool.init")) {
- executionTool.prepareForExecution(request.getId(), builtTargets, builtAspects);
+ executionTool.prepareForExecution(
+ request.getId(),
+ builtTargets,
+ builtAspects,
+ loadingResult.getNotSymlinkedInExecrootDirectories());
}
// TODO(b/199053098): implement support for --nobuild.
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
index 5f2969b..1283001 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -247,7 +247,10 @@
* tests for these setup steps.
*/
public void prepareForExecution(
- UUID buildId, Set<ConfiguredTargetKey> builtTargets, Set<AspectKey> builtAspects)
+ UUID buildId,
+ Set<ConfiguredTargetKey> builtTargets,
+ Set<AspectKey> builtAspects,
+ ImmutableSortedSet<String> notSymlinkedInExecrootDirectories)
throws AbruptExitException, BuildFailedException, InterruptedException {
init();
BuildRequestOptions options = request.getBuildOptions();
@@ -260,12 +263,33 @@
"--experimental_merged_skyframe_analysis_execution requires a single package path entry."
+ " Found a list of size: %s",
pkgPathEntries.size());
- Root singleSourceRoot = Iterables.getOnlyElement(pkgPathEntries);
- PackageRoots noSymlinkPackageRoots = new PackageRootsNoSymlinkCreation(singleSourceRoot);
- env.getEventBus().post(new ExecRootPreparedEvent(noSymlinkPackageRoots.getPackageRootsMap()));
- env.getSkyframeBuildView()
- .getArtifactFactory()
- .setPackageRoots(noSymlinkPackageRoots.getPackageRootLookup());
+
+ try (SilentCloseable c = Profiler.instance().profile("preparingExecroot")) {
+ Root singleSourceRoot = Iterables.getOnlyElement(pkgPathEntries);
+ PackageRoots noSymlinkPackageRoots = new PackageRootsNoSymlinkCreation(singleSourceRoot);
+ try {
+ SymlinkForest.eagerlyPlantSymlinkForestSinglePackagePath(
+ getExecRoot(),
+ singleSourceRoot.asPath(),
+ /*prefix=*/ env.getDirectories().getProductName() + "-",
+ notSymlinkedInExecrootDirectories,
+ request.getOptions(BuildLanguageOptions.class).experimentalSiblingRepositoryLayout);
+ } catch (IOException e) {
+ throw new AbruptExitException(
+ DetailedExitCode.of(
+ FailureDetail.newBuilder()
+ .setMessage("Failed to prepare the symlink forest")
+ .setSymlinkForest(
+ FailureDetails.SymlinkForest.newBuilder()
+ .setCode(FailureDetails.SymlinkForest.Code.CREATION_FAILED))
+ .build()),
+ e);
+ }
+ env.getEventBus().post(new ExecRootPreparedEvent(noSymlinkPackageRoots.getPackageRootsMap()));
+ env.getSkyframeBuildView()
+ .getArtifactFactory()
+ .setPackageRoots(noSymlinkPackageRoots.getPackageRootLookup());
+ }
OutputService outputService = env.getOutputService();
ModifiedFileSet modifiedOutputFiles = ModifiedFileSet.EVERYTHING_MODIFIED;
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/SymlinkForest.java b/src/main/java/com/google/devtools/build/lib/buildtool/SymlinkForest.java
index 9a70756..5b948dd 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/SymlinkForest.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/SymlinkForest.java
@@ -112,7 +112,9 @@
*/
@VisibleForTesting
@ThreadSafety.ThreadSafe
- void deleteTreesBelowNotPrefixed(Path dir, String prefix) throws IOException {
+ static void deleteTreesBelowNotPrefixed(
+ Path dir, String prefix, ImmutableSortedSet<String> notSymlinkedInExecrootDirectories)
+ throws IOException {
for (Path p : dir.getDirectoryEntries()) {
@@ -337,7 +339,7 @@
* @return the symlinks that have been planted
*/
public ImmutableList<Path> plantSymlinkForest() throws IOException, AbruptExitException {
- deleteTreesBelowNotPrefixed(execroot, prefix);
+ deleteTreesBelowNotPrefixed(execroot, prefix, notSymlinkedInExecrootDirectories);
if (siblingRepositoryLayout) {
// Delete execroot/../<symlinks> to directories representing external repositories.
@@ -419,6 +421,38 @@
return plantedSymlinks.build();
}
+ /**
+ * Eagerly plant the symlinks from execroot to the source root provided by the single package path
+ * of the current build. Only works with a single package path. Before planting the new symlinks,
+ * remove all existing symlinks in execroot which don't match certain criteria.
+ */
+ public static void eagerlyPlantSymlinkForestSinglePackagePath(
+ Path execroot,
+ Path sourceRoot,
+ String prefix,
+ ImmutableSortedSet<String> notSymlinkedInExecrootDirectories,
+ boolean siblingRepositoryLayout)
+ throws IOException {
+ deleteTreesBelowNotPrefixed(execroot, prefix, notSymlinkedInExecrootDirectories);
+
+ // Plant everything under the single source root.
+ for (Path target : sourceRoot.getDirectoryEntries()) {
+
+ String baseName = target.getBaseName();
+ if (notSymlinkedInExecrootDirectories.contains(baseName)) {
+ continue;
+ }
+ Path execPath = execroot.getRelative(baseName);
+ // Create any links that don't start with bazel-, and ignore external/ directory if
+ // user has it in the source tree because it conflicts with external repository location.
+ if (!baseName.startsWith(prefix)
+ && (siblingRepositoryLayout
+ || !baseName.equals(LabelConstants.EXTERNAL_PATH_PREFIX.getBaseName()))) {
+ execPath.createSymbolicLink(target);
+ }
+ }
+ }
+
private static DetailedExitCode detailedSymlinkForestExitCode(String message, Code code) {
return DetailedExitCode.of(
FailureDetail.newBuilder()
diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/BUILD b/src/test/java/com/google/devtools/build/lib/buildtool/BUILD
index 706a54c..b3028ad 100644
--- a/src/test/java/com/google/devtools/build/lib/buildtool/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/buildtool/BUILD
@@ -640,8 +640,7 @@
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/analysis:view_creation_failed_exception",
- "//src/main/java/com/google/devtools/build/lib/util/io",
- "//src/main/protobuf:failure_details_java_proto",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
"//src/test/java/com/google/devtools/build/lib/buildtool/util",
"//third_party:guava",
"//third_party:junit4",
diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/SkymeldBuildIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/buildtool/SkymeldBuildIntegrationTest.java
index bc2a0be..406b46a 100644
--- a/src/test/java/com/google/devtools/build/lib/buildtool/SkymeldBuildIntegrationTest.java
+++ b/src/test/java/com/google/devtools/build/lib/buildtool/SkymeldBuildIntegrationTest.java
@@ -14,14 +14,13 @@
package com.google.devtools.build.lib.buildtool;
import static com.google.common.truth.Truth.assertThat;
-import static com.google.devtools.build.lib.server.FailureDetails.Spawn.Code.NON_ZERO_EXIT;
import static org.junit.Assert.assertThrows;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.BuildFailedException;
import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
import com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase;
-import com.google.devtools.build.lib.util.io.RecordingOutErr;
+import com.google.devtools.build.lib.vfs.Path;
import com.google.testing.junit.testparameterinjector.TestParameter;
import com.google.testing.junit.testparameterinjector.TestParameterInjector;
import java.io.IOException;
@@ -153,7 +152,7 @@
}
@Test
- public void noSymlinkPlantedLocalAction_failureNoSuchFileOrDirectory() throws Exception {
+ public void symlinkPlantedLocalAction_success() throws Exception {
addOptions("--spawn_strategy=standalone");
write(
"foo/BUILD",
@@ -161,21 +160,64 @@
" name = 'foo',",
" srcs = ['foo.in'],",
" outs = ['foo.out'],",
- " cmd = 'cp foo.in $(location foo.out)'",
+ " cmd = 'cp $< $@'",
")");
write("foo/foo.in");
- outErr = new RecordingOutErr();
- BuildFailedException e =
- assertThrows(BuildFailedException.class, () -> buildTarget("//foo:foo"));
- String err = ((RecordingOutErr) outErr).errAsLatin1();
+ BuildResult result = buildTarget("//foo:foo");
- assertThat(e.getDetailedExitCode().getFailureDetail().getSpawn().getCode())
- .isEqualTo(NON_ZERO_EXIT);
- assertThat(err)
- .contains(
- "Executing genrule //foo:foo failed: (Exit 1): bash failed: error executing command"
- + " (from target //foo:foo)");
- assertThat(err).contains("No such file or directory");
+ assertThat(result.getSuccess()).isTrue();
+ assertSingleOutputBuilt("//foo:foo");
+ }
+
+ @Test
+ public void symlinksPlanted() throws Exception {
+ Path execroot = directories.getExecRoot(directories.getWorkspace().getBaseName());
+ writeMyRuleBzl();
+ Path fooDir =
+ write(
+ "foo/BUILD",
+ "load('//foo:my_rule.bzl', 'my_rule')",
+ "my_rule(name = 'foo', srcs = ['foo.in'])")
+ .getParentDirectory();
+ write("foo/foo.in");
+ Path unusedDir = write("unused/dummy").getParentDirectory();
+
+ // Before the build: no symlink.
+ assertThat(execroot.getRelative("foo").exists()).isFalse();
+
+ buildTarget("//foo:foo");
+
+ // After the build: symlinks to the source directory, even unused packages.
+ assertThat(execroot.getRelative("foo").resolveSymbolicLinks()).isEqualTo(fooDir);
+ assertThat(execroot.getRelative("unused").resolveSymbolicLinks()).isEqualTo(unusedDir);
+ }
+
+ @Test
+ public void symlinksReplantedEachBuild() throws Exception {
+ Path execroot = directories.getExecRoot(directories.getWorkspace().getBaseName());
+ writeMyRuleBzl();
+ Path fooDir =
+ write(
+ "foo/BUILD",
+ "load('//foo:my_rule.bzl', 'my_rule')",
+ "my_rule(name = 'foo', srcs = ['foo.in'])")
+ .getParentDirectory();
+ write("foo/foo.in");
+ Path unusedDir = write("unused/dummy").getParentDirectory();
+
+ buildTarget("//foo:foo");
+
+ // After the 1st build: symlinks to the source directory, even unused packages.
+ assertThat(execroot.getRelative("foo").resolveSymbolicLinks()).isEqualTo(fooDir);
+ assertThat(execroot.getRelative("unused").resolveSymbolicLinks()).isEqualTo(unusedDir);
+
+ unusedDir.deleteTree();
+
+ buildTarget("//foo:foo");
+
+ // After the 2nd build: symlink to unusedDir is gone, since the package itself was deleted.
+ assertThat(execroot.getRelative("foo").resolveSymbolicLinks()).isEqualTo(fooDir);
+ assertThat(execroot.getRelative("unused").exists()).isFalse();
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/SymlinkForestTest.java b/src/test/java/com/google/devtools/build/lib/buildtool/SymlinkForestTest.java
index d42ad82..4df48ae 100644
--- a/src/test/java/com/google/devtools/build/lib/buildtool/SymlinkForestTest.java
+++ b/src/test/java/com/google/devtools/build/lib/buildtool/SymlinkForestTest.java
@@ -138,7 +138,8 @@
@Test
public void testDeleteTreesBelowNotPrefixed() throws IOException {
createTestDirectoryTree();
- new SymlinkForest(ImmutableMap.of(), topDir, "").deleteTreesBelowNotPrefixed(topDir, "file-");
+ SymlinkForest.deleteTreesBelowNotPrefixed(
+ topDir, "file-", /*notSymlinkedInExecrootDirectories=*/ ImmutableSortedSet.of());
assertThat(file1.exists()).isTrue();
assertThat(file2.exists()).isTrue();
assertThat(aDir.exists()).isFalse();
