Symlink creation: add flag for in-process creation
This is in preparation for skipping manifest creation. If we don't have
an input manifest and no output service, we can't fall back to the
subprocess method, which requires a manifest. Instead, we will fall back
to an in-process implementation. This should be rare.
This also provides a fallback mechanism if the embedded tool is
unavailable, and we may switch this on by default if the performance is
acceptable.
PiperOrigin-RevId: 282895566
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java
index bd294cf..3b4583f 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java
@@ -45,6 +45,7 @@
private final Artifact outputManifest;
private final boolean filesetTree;
private final boolean enableRunfiles;
+ private final boolean inprocessSymlinkCreation;
/**
* Creates SymlinkTreeAction instance.
@@ -71,7 +72,8 @@
outputManifest,
filesetTree,
config.getActionEnvironment(),
- config.runfilesEnabled());
+ config.runfilesEnabled(),
+ config.inprocessSymlinkCreation());
}
/**
@@ -94,7 +96,8 @@
Artifact outputManifest,
boolean filesetTree,
ActionEnvironment env,
- boolean enableRunfiles) {
+ boolean enableRunfiles,
+ boolean inprocessSymlinkCreation) {
super(owner, ImmutableList.of(inputManifest), ImmutableList.of(outputManifest), env);
Preconditions.checkArgument(outputManifest.getPath().getBaseName().equals("MANIFEST"));
Preconditions.checkArgument(
@@ -104,6 +107,7 @@
this.outputManifest = outputManifest;
this.filesetTree = filesetTree;
this.enableRunfiles = enableRunfiles;
+ this.inprocessSymlinkCreation = inprocessSymlinkCreation;
}
public Artifact getInputManifest() {
@@ -127,6 +131,10 @@
return enableRunfiles;
}
+ public boolean inprocessSymlinkCreation() {
+ return inprocessSymlinkCreation;
+ }
+
@Override
public String getMnemonic() {
return "SymlinkTree";
@@ -143,6 +151,7 @@
fp.addString(GUID);
fp.addBoolean(filesetTree);
fp.addBoolean(enableRunfiles);
+ fp.addBoolean(inprocessSymlinkCreation);
env.addTo(fp);
// We need to ensure that the fingerprints for two different instances of this action are
// different. Consider the hypothetical scenario where we add a second runfiles object to this
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
index 54fa49c..979e3c3 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
@@ -857,6 +857,10 @@
return runfilesEnabled(this.options);
}
+ public boolean inprocessSymlinkCreation() {
+ return options.inprocessSymlinkCreation;
+ }
+
/**
* Returns a modified copy of {@code executionInfo} if any {@code executionInfoModifiers} apply to
* the given {@code mnemonic}. Otherwise returns {@code executionInfo} unchanged.
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java
index b20e037..4920729 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java
@@ -23,6 +23,7 @@
import com.google.devtools.build.lib.util.RegexFilter;
import com.google.devtools.common.options.Converter;
import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Converters.BooleanConverter;
import com.google.devtools.common.options.EnumConverter;
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionDefinition;
@@ -789,6 +790,16 @@
+ "build")
public IncludeConfigFragmentsEnum includeRequiredConfigFragmentsProvider;
+ @Option(
+ name = "experimental_inprocess_symlink_creation",
+ defaultValue = "false",
+ converter = BooleanConverter.class,
+ documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
+ metadataTags = OptionMetadataTag.EXPERIMENTAL,
+ effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS, OptionEffectTag.EXECUTION},
+ help = "Whether to make direct file system calls to create symlink trees")
+ public boolean inprocessSymlinkCreation;
+
/** Ways configured targets may provide the {@link BuildConfiguration.Fragment}s they require. */
public enum IncludeConfigFragmentsEnum {
// Don't offer the provider at all. This is best for most builds, which don't use this
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java
index e5e2124..5aefaa4 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java
@@ -18,6 +18,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.shell.Command;
@@ -33,8 +34,10 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* Helper class responsible for the symlink tree creation. Used to generate runfiles and fileset
@@ -66,6 +69,25 @@
return symlinkTreeRoot;
}
+ /** Creates a symlink tree using direct system calls. */
+ public void createSymlinksDirectly(Path symlinkTreeRoot, Map<PathFragment, Artifact> symlinks)
+ throws IOException {
+ Preconditions.checkState(!filesetTree);
+ Set<Path> dirs = new HashSet<>();
+ for (Map.Entry<PathFragment, Artifact> e : symlinks.entrySet()) {
+ Path symlinkPath = symlinkTreeRoot.getRelative(e.getKey());
+ Path parentDirectory = symlinkPath.getParentDirectory();
+ if (dirs.add(parentDirectory)) {
+ parentDirectory.createDirectoryAndParents();
+ }
+ if (e.getValue() == null) {
+ FileSystemUtils.createEmptyFile(symlinkPath);
+ } else {
+ symlinkPath.createSymbolicLink(e.getValue().getPath().asFragment());
+ }
+ }
+ }
+
/**
* Creates symlink tree and output manifest using the {@code build-runfiles.cc} tool.
*
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
index 4256c4a..c71fce0 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
@@ -107,6 +107,20 @@
}
} else if (!action.isRunfilesEnabled()) {
createSymlinkTreeHelper(action, actionExecutionContext).copyManifest();
+ } else if (action.inprocessSymlinkCreation() && !action.isFilesetTree()) {
+ try {
+ createSymlinkTreeHelper(action, actionExecutionContext)
+ .createSymlinksDirectly(
+ action.getOutputManifest().getPath().getParentDirectory(),
+ action
+ .getRunfiles()
+ .getRunfilesInputs(
+ actionExecutionContext.getEventHandler(),
+ action.getOwner().getLocation(),
+ actionExecutionContext.getPathResolver()));
+ } catch (IOException e) {
+ throw new EnvironmentalExecException(e).toActionExecutionException(action);
+ }
} else {
Map<String, String> resolvedEnv = new LinkedHashMap<>();
action.getEnvironment().resolve(resolvedEnv, actionExecutionContext.getClientEnv());
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeActionTest.java b/src/test/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeActionTest.java
index b0b7320..d6afec3 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeActionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeActionTest.java
@@ -35,6 +35,7 @@
private enum KeyAttributes {
FILESET,
RUNFILES_FLAG,
+ INPROCESS,
FIXED_ENVIRONMENT,
VARIABLE_ENVIRONMENT
}
@@ -52,6 +53,7 @@
public Action generate(ImmutableSet<KeyAttributes> attributesToFlip) {
boolean filesetTree = attributesToFlip.contains(KeyAttributes.FILESET);
boolean enableRunfiles = attributesToFlip.contains(KeyAttributes.RUNFILES_FLAG);
+ boolean inprocessSymlinkCreation = attributesToFlip.contains(KeyAttributes.INPROCESS);
ActionEnvironment env =
ActionEnvironment.create(
@@ -77,7 +79,8 @@
outputManifest,
filesetTree,
env,
- enableRunfiles);
+ enableRunfiles,
+ inprocessSymlinkCreation);
}
},
actionKeyContext);
diff --git a/src/test/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategyTest.java b/src/test/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategyTest.java
index b355035..9bb02cc 100644
--- a/src/test/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategyTest.java
+++ b/src/test/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategyTest.java
@@ -15,9 +15,11 @@
package com.google.devtools.build.lib.exec;
import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -33,8 +35,11 @@
import com.google.devtools.build.lib.analysis.actions.SymlinkTreeActionContext;
import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.OutputService;
+import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Symlinks;
import java.util.Map;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -66,7 +71,7 @@
when(outputService.canCreateSymlinkTree()).thenReturn(true);
Artifact inputManifest = getBinArtifactWithNoOwner("dir/manifest.in");
- Artifact outputManifest = getBinArtifactWithNoOwner("dir/MANIFEST");
+ Artifact outputManifest = getBinArtifactWithNoOwner("dir.runfiles/MANIFEST");
Artifact runfile = getBinArtifactWithNoOwner("dir/runfile");
doAnswer(
(i) -> {
@@ -89,7 +94,8 @@
outputManifest,
/*filesetTree=*/ false,
ActionEnvironment.EMPTY,
- /*enableRunfiles=*/ true);
+ /*enableRunfiles=*/ true,
+ /*inprocessSymlinkCreation=*/ false);
action.execute(context);
@@ -103,4 +109,49 @@
PathFragment.create("TESTING/dir/empty"),
null);
}
+
+ @Test
+ public void inprocessSymlinkCreation() throws Exception {
+ ActionExecutionContext context = mock(ActionExecutionContext.class);
+ OutputService outputService = mock(OutputService.class);
+ StoredEventHandler eventHandler = new StoredEventHandler();
+
+ when(context.getContext(SymlinkTreeActionContext.class))
+ .thenReturn(new SymlinkTreeStrategy(outputService, null));
+ when(context.getInputPath(any())).thenAnswer((i) -> ((Artifact) i.getArgument(0)).getPath());
+ when(context.getEventHandler()).thenReturn(eventHandler);
+ when(outputService.canCreateSymlinkTree()).thenReturn(false);
+
+ Artifact inputManifest = getBinArtifactWithNoOwner("dir/manifest.in");
+ Artifact outputManifest = getBinArtifactWithNoOwner("dir.runfiles/MANIFEST");
+ Artifact runfile = getBinArtifactWithNoOwner("dir/runfile");
+
+ Runfiles runfiles =
+ new Runfiles.Builder("TESTING", false)
+ .setEmptyFilesSupplier((paths) -> ImmutableList.of(PathFragment.create("dir/empty")))
+ .addArtifact(runfile)
+ .build();
+ SymlinkTreeAction action =
+ new SymlinkTreeAction(
+ ActionsTestUtil.NULL_ACTION_OWNER,
+ inputManifest,
+ runfiles,
+ outputManifest,
+ /*filesetTree=*/ false,
+ ActionEnvironment.EMPTY,
+ /*enableRunfiles=*/ true,
+ /*inprocessSymlinkCreation=*/ true);
+
+ action.execute(context);
+ // Check that the OutputService is not used.
+ verify(outputService, never()).createSymlinkTree(any(), any());
+
+ Path p = outputManifest.getPath().getParentDirectory().getRelative("TESTING/dir/runfile");
+ assertWithMessage("Path %s expected to exist", p).that(p.exists(Symlinks.NOFOLLOW)).isTrue();
+ assertWithMessage("Path %s expected to be a symlink", p).that(p.isSymbolicLink()).isTrue();
+ assertThat(p.readSymbolicLink()).isEqualTo(runfile.getPath().asFragment());
+ Path q = outputManifest.getPath().getParentDirectory().getRelative("TESTING/dir/empty");
+ assertWithMessage("Path %s expected to be a file", q).that(q.isFile()).isTrue();
+ assertThat(FileSystemUtils.readContent(q)).isEmpty();
+ }
}
