Symlink creation: add flag for in-process creation
This is in preparation for skipping manifest creation. If we don't have
an input manifest and no output service, we can't fall back to the
subprocess method, which requires a manifest. Instead, we will fall back
to an in-process implementation. This should be rare.
This also provides a fallback mechanism if the embedded tool is
unavailable, and we may switch this on by default if the performance is
acceptable.
PiperOrigin-RevId: 282895566
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java
index bd294cf..3b4583f 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java
@@ -45,6 +45,7 @@
private final Artifact outputManifest;
private final boolean filesetTree;
private final boolean enableRunfiles;
+ private final boolean inprocessSymlinkCreation;
/**
* Creates SymlinkTreeAction instance.
@@ -71,7 +72,8 @@
outputManifest,
filesetTree,
config.getActionEnvironment(),
- config.runfilesEnabled());
+ config.runfilesEnabled(),
+ config.inprocessSymlinkCreation());
}
/**
@@ -94,7 +96,8 @@
Artifact outputManifest,
boolean filesetTree,
ActionEnvironment env,
- boolean enableRunfiles) {
+ boolean enableRunfiles,
+ boolean inprocessSymlinkCreation) {
super(owner, ImmutableList.of(inputManifest), ImmutableList.of(outputManifest), env);
Preconditions.checkArgument(outputManifest.getPath().getBaseName().equals("MANIFEST"));
Preconditions.checkArgument(
@@ -104,6 +107,7 @@
this.outputManifest = outputManifest;
this.filesetTree = filesetTree;
this.enableRunfiles = enableRunfiles;
+ this.inprocessSymlinkCreation = inprocessSymlinkCreation;
}
public Artifact getInputManifest() {
@@ -127,6 +131,10 @@
return enableRunfiles;
}
+ public boolean inprocessSymlinkCreation() {
+ return inprocessSymlinkCreation;
+ }
+
@Override
public String getMnemonic() {
return "SymlinkTree";
@@ -143,6 +151,7 @@
fp.addString(GUID);
fp.addBoolean(filesetTree);
fp.addBoolean(enableRunfiles);
+ fp.addBoolean(inprocessSymlinkCreation);
env.addTo(fp);
// We need to ensure that the fingerprints for two different instances of this action are
// different. Consider the hypothetical scenario where we add a second runfiles object to this
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
index 54fa49c..979e3c3 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
@@ -857,6 +857,10 @@
return runfilesEnabled(this.options);
}
+ public boolean inprocessSymlinkCreation() {
+ return options.inprocessSymlinkCreation;
+ }
+
/**
* Returns a modified copy of {@code executionInfo} if any {@code executionInfoModifiers} apply to
* the given {@code mnemonic}. Otherwise returns {@code executionInfo} unchanged.
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java
index b20e037..4920729 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java
@@ -23,6 +23,7 @@
import com.google.devtools.build.lib.util.RegexFilter;
import com.google.devtools.common.options.Converter;
import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Converters.BooleanConverter;
import com.google.devtools.common.options.EnumConverter;
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionDefinition;
@@ -789,6 +790,16 @@
+ "build")
public IncludeConfigFragmentsEnum includeRequiredConfigFragmentsProvider;
+ @Option(
+ name = "experimental_inprocess_symlink_creation",
+ defaultValue = "false",
+ converter = BooleanConverter.class,
+ documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
+ metadataTags = OptionMetadataTag.EXPERIMENTAL,
+ effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS, OptionEffectTag.EXECUTION},
+ help = "Whether to make direct file system calls to create symlink trees")
+ public boolean inprocessSymlinkCreation;
+
/** Ways configured targets may provide the {@link BuildConfiguration.Fragment}s they require. */
public enum IncludeConfigFragmentsEnum {
// Don't offer the provider at all. This is best for most builds, which don't use this
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java
index e5e2124..5aefaa4 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java
@@ -18,6 +18,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.shell.Command;
@@ -33,8 +34,10 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* Helper class responsible for the symlink tree creation. Used to generate runfiles and fileset
@@ -66,6 +69,25 @@
return symlinkTreeRoot;
}
+ /** Creates a symlink tree using direct system calls. */
+ public void createSymlinksDirectly(Path symlinkTreeRoot, Map<PathFragment, Artifact> symlinks)
+ throws IOException {
+ Preconditions.checkState(!filesetTree);
+ Set<Path> dirs = new HashSet<>();
+ for (Map.Entry<PathFragment, Artifact> e : symlinks.entrySet()) {
+ Path symlinkPath = symlinkTreeRoot.getRelative(e.getKey());
+ Path parentDirectory = symlinkPath.getParentDirectory();
+ if (dirs.add(parentDirectory)) {
+ parentDirectory.createDirectoryAndParents();
+ }
+ if (e.getValue() == null) {
+ FileSystemUtils.createEmptyFile(symlinkPath);
+ } else {
+ symlinkPath.createSymbolicLink(e.getValue().getPath().asFragment());
+ }
+ }
+ }
+
/**
* Creates symlink tree and output manifest using the {@code build-runfiles.cc} tool.
*
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
index 4256c4a..c71fce0 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
@@ -107,6 +107,20 @@
}
} else if (!action.isRunfilesEnabled()) {
createSymlinkTreeHelper(action, actionExecutionContext).copyManifest();
+ } else if (action.inprocessSymlinkCreation() && !action.isFilesetTree()) {
+ try {
+ createSymlinkTreeHelper(action, actionExecutionContext)
+ .createSymlinksDirectly(
+ action.getOutputManifest().getPath().getParentDirectory(),
+ action
+ .getRunfiles()
+ .getRunfilesInputs(
+ actionExecutionContext.getEventHandler(),
+ action.getOwner().getLocation(),
+ actionExecutionContext.getPathResolver()));
+ } catch (IOException e) {
+ throw new EnvironmentalExecException(e).toActionExecutionException(action);
+ }
} else {
Map<String, String> resolvedEnv = new LinkedHashMap<>();
action.getEnvironment().resolve(resolvedEnv, actionExecutionContext.getClientEnv());
