Allow skipping creation of the runfiles manifest
If --experimental_skip_runfiles_manifests is enabled, Bazel does not
write a <rule>.runfiles_manifest file for binary and test rules, and
instead creates the symlink tree from the in-memory representation.
The way this is implemented is by making the symlink tree action not
depend on the input manifest. If there are other dependencies on the
manifest, it will still be written. I have carefully audited the code,
and I believe that there are no such dependencies, and the manifest is
generally not part of the providers published by a configured target.
Note that this has no effect on Fileset manifests, on Windows, and if
--experimental_enable_runfiles is false.
RELNOTES: We are planning to deprecate the runfiles manifest files, which aren't safe in the presence of whitespace, and also unnecessarily require local CPU when remote execution is used. This release adds --experimental_skip_runfiles_manifests to disable the generation of the input manifests (rule.manifest files) in most cases. Note that this flag has no effect on Windows by default or if --experimental_enable_runfiles is explicitly set to false.
PiperOrigin-RevId: 283078443
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java
index 3b4583f..adde535 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeAction.java
@@ -40,12 +40,13 @@
private static final String GUID = "63412bda-4026-4c8e-a3ad-7deb397728d4";
- private final Artifact inputManifest;
+ @Nullable private final Artifact inputManifest;
private final Runfiles runfiles;
private final Artifact outputManifest;
private final boolean filesetTree;
private final boolean enableRunfiles;
private final boolean inprocessSymlinkCreation;
+ private final boolean skipRunfilesManifests;
/**
* Creates SymlinkTreeAction instance.
@@ -73,7 +74,8 @@
filesetTree,
config.getActionEnvironment(),
config.runfilesEnabled(),
- config.inprocessSymlinkCreation());
+ config.inprocessSymlinkCreation(),
+ config.skipRunfilesManifests());
}
/**
@@ -97,17 +99,26 @@
boolean filesetTree,
ActionEnvironment env,
boolean enableRunfiles,
- boolean inprocessSymlinkCreation) {
- super(owner, ImmutableList.of(inputManifest), ImmutableList.of(outputManifest), env);
+ boolean inprocessSymlinkCreation,
+ boolean skipRunfilesManifests) {
+ super(
+ owner,
+ skipRunfilesManifests && enableRunfiles && !filesetTree
+ ? ImmutableList.of()
+ : ImmutableList.of(inputManifest),
+ ImmutableList.of(outputManifest),
+ env);
Preconditions.checkArgument(outputManifest.getPath().getBaseName().equals("MANIFEST"));
Preconditions.checkArgument(
(runfiles == null) == filesetTree, "Runfiles must be null iff this is a fileset action");
- this.inputManifest = inputManifest;
+ this.inputManifest =
+ skipRunfilesManifests && enableRunfiles && !filesetTree ? null : inputManifest;
this.runfiles = runfiles;
this.outputManifest = outputManifest;
this.filesetTree = filesetTree;
this.enableRunfiles = enableRunfiles;
this.inprocessSymlinkCreation = inprocessSymlinkCreation;
+ this.skipRunfilesManifests = skipRunfilesManifests && enableRunfiles && !filesetTree;
}
public Artifact getInputManifest() {
@@ -135,6 +146,10 @@
return inprocessSymlinkCreation;
}
+ public boolean skipRunfilesManifests() {
+ return skipRunfilesManifests;
+ }
+
@Override
public String getMnemonic() {
return "SymlinkTree";
@@ -152,6 +167,7 @@
fp.addBoolean(filesetTree);
fp.addBoolean(enableRunfiles);
fp.addBoolean(inprocessSymlinkCreation);
+ fp.addBoolean(skipRunfilesManifests);
env.addTo(fp);
// We need to ensure that the fingerprints for two different instances of this action are
// different. Consider the hypothetical scenario where we add a second runfiles object to this
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
index 979e3c3..ba0c133 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
@@ -861,6 +861,10 @@
return options.inprocessSymlinkCreation;
}
+ public boolean skipRunfilesManifests() {
+ return options.skipRunfilesManifests;
+ }
+
/**
* Returns a modified copy of {@code executionInfo} if any {@code executionInfoModifiers} apply to
* the given {@code mnemonic}. Otherwise returns {@code executionInfo} unchanged.
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java
index 4920729..8b3235d 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java
@@ -800,6 +800,23 @@
help = "Whether to make direct file system calls to create symlink trees")
public boolean inprocessSymlinkCreation;
+ @Option(
+ name = "experimental_skip_runfiles_manifests",
+ defaultValue = "false",
+ converter = BooleanConverter.class,
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ metadataTags = OptionMetadataTag.HIDDEN,
+ effectTags = {
+ OptionEffectTag.LOADING_AND_ANALYSIS,
+ OptionEffectTag.EXECUTION,
+ OptionEffectTag.LOSES_INCREMENTAL_STATE,
+ OptionEffectTag.AFFECTS_OUTPUTS
+ },
+ help =
+ "If enabled, Bazel does not create runfiles symlink manifests. This flag is ignored "
+ + "if --experimental_enable_runfiles is set to false.")
+ public boolean skipRunfilesManifests;
+
/** Ways configured targets may provide the {@link BuildConfiguration.Fragment}s they require. */
public enum IncludeConfigFragmentsEnum {
// Don't offer the provider at all. This is best for most builds, which don't use this
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
index c71fce0..fd37c68 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
@@ -27,6 +27,8 @@
import com.google.devtools.build.lib.analysis.actions.SymlinkTreeAction;
import com.google.devtools.build.lib.analysis.actions.SymlinkTreeActionContext;
import com.google.devtools.build.lib.profiler.AutoProfiler;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.OutputService;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
@@ -65,7 +67,10 @@
"running " + action.prettyPrint(), logger, /*minTimeForLoggingInMilliseconds=*/ 100)) {
try {
if (outputService != null && outputService.canCreateSymlinkTree()) {
- Path inputManifest = actionExecutionContext.getInputPath(action.getInputManifest());
+ Path inputManifest =
+ action.getInputManifest() == null
+ ? null
+ : actionExecutionContext.getInputPath(action.getInputManifest());
Map<PathFragment, PathFragment> symlinks;
if (action.getRunfiles() != null) {
try {
@@ -83,6 +88,7 @@
}
} else {
Preconditions.checkState(action.isFilesetTree());
+ Preconditions.checkNotNull(inputManifest);
try {
symlinks = SymlinkTreeHelper.readSymlinksFromFilesetManifest(inputManifest);
} catch (IOException e) {
@@ -95,19 +101,34 @@
symlinks,
action.getOutputManifest().getExecPath().getParentDirectory());
- // Link output manifest on success. We avoid a file copy as these manifests may be large.
- // Note that this step has to come last because the OutputService may delete any
- // pre-existing symlink tree before creating a new one.
Path outputManifest = actionExecutionContext.getInputPath(action.getOutputManifest());
- try {
- outputManifest.createSymbolicLink(inputManifest);
- } catch (IOException e) {
- throw new EnvironmentalExecException(
- "Failed to link output manifest '" + outputManifest.getPathString() + "'", e);
+ if (inputManifest == null) {
+ // If we don't have an input manifest, then create a file containing a fingerprint of
+ // the runfiles object.
+ Fingerprint fp = new Fingerprint();
+ action.getRunfiles().fingerprint(fp);
+ String hexDigest = fp.hexDigestAndReset();
+ try {
+ FileSystemUtils.writeContentAsLatin1(outputManifest, hexDigest);
+ } catch (IOException e) {
+ throw new EnvironmentalExecException(
+ "Failed to link output manifest '" + outputManifest.getPathString() + "'", e);
+ }
+ } else {
+ // Link output manifest on success. We avoid a file copy as these manifests may be
+ // large. Note that this step has to come last because the OutputService may delete any
+ // pre-existing symlink tree before creating a new one.
+ try {
+ outputManifest.createSymbolicLink(inputManifest);
+ } catch (IOException e) {
+ throw new EnvironmentalExecException(
+ "Failed to link output manifest '" + outputManifest.getPathString() + "'", e);
+ }
}
} else if (!action.isRunfilesEnabled()) {
createSymlinkTreeHelper(action, actionExecutionContext).copyManifest();
- } else if (action.inprocessSymlinkCreation() && !action.isFilesetTree()) {
+ } else if (action.getInputManifest() == null
+ || (action.inprocessSymlinkCreation() && !action.isFilesetTree())) {
try {
createSymlinkTreeHelper(action, actionExecutionContext)
.createSymlinksDirectly(
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeActionTest.java b/src/test/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeActionTest.java
index beb333c..1b01f43 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeActionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/actions/SymlinkTreeActionTest.java
@@ -21,6 +21,7 @@
import com.google.devtools.build.lib.analysis.Runfiles;
import com.google.devtools.build.lib.analysis.util.ActionTester;
import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -43,6 +44,13 @@
VARIABLE_ENVIRONMENT
}
+ private enum SkipManifestAttributes {
+ RUNFILES,
+ INPROCESS_SYMLINKS,
+ FIXED_ENVIRONMENT,
+ VARIABLE_ENVIRONMENT
+ }
+
@Test
public void testComputeKey() throws Exception {
final Artifact inputManifest = getBinArtifactWithNoOwner("dir/manifest.in");
@@ -66,7 +74,8 @@
attributesToFlip.contains(RunfilesActionAttributes.FIXED_ENVIRONMENT),
attributesToFlip.contains(RunfilesActionAttributes.VARIABLE_ENVIRONMENT)),
attributesToFlip.contains(RunfilesActionAttributes.ENABLE_RUNFILES),
- attributesToFlip.contains(RunfilesActionAttributes.INPROCESS_SYMLINKS)))
+ attributesToFlip.contains(RunfilesActionAttributes.INPROCESS_SYMLINKS),
+ /*skipRunfilesManifests=*/ false))
.combinations(
FilesetActionAttributes.class,
(attributesToFlip) ->
@@ -80,7 +89,26 @@
attributesToFlip.contains(FilesetActionAttributes.FIXED_ENVIRONMENT),
attributesToFlip.contains(FilesetActionAttributes.VARIABLE_ENVIRONMENT)),
attributesToFlip.contains(FilesetActionAttributes.ENABLE_RUNFILES),
- attributesToFlip.contains(FilesetActionAttributes.INPROCESS_SYMLINKS)))
+ attributesToFlip.contains(FilesetActionAttributes.INPROCESS_SYMLINKS),
+ /*skipRunfilesManifests=*/ false))
+ .combinations(
+ SkipManifestAttributes.class,
+ (attributesToFlip) ->
+ // skipRunfilesManifests requires !filesetTree and enableRunfiles
+ new SymlinkTreeAction(
+ ActionsTestUtil.NULL_ACTION_OWNER,
+ inputManifest,
+ attributesToFlip.contains(SkipManifestAttributes.RUNFILES)
+ ? new Runfiles.Builder("TESTING", false).addArtifact(runfile).build()
+ : new Runfiles.Builder("TESTING", false).addArtifact(runfile2).build(),
+ outputManifest,
+ /*filesetTree=*/ false,
+ createActionEnvironment(
+ attributesToFlip.contains(SkipManifestAttributes.FIXED_ENVIRONMENT),
+ attributesToFlip.contains(SkipManifestAttributes.VARIABLE_ENVIRONMENT)),
+ /*enableRunfiles=*/ true,
+ attributesToFlip.contains(SkipManifestAttributes.INPROCESS_SYMLINKS),
+ /*skipRunfilesManifests=*/ true))
.runTest();
}
@@ -89,4 +117,23 @@
fixed ? ImmutableMap.of("a", "b") : ImmutableMap.of(),
variable ? ImmutableSet.of("c") : ImmutableSet.of());
}
+
+ @Test
+ public void testNullRunfilesThrows() {
+ Artifact inputManifest = getBinArtifactWithNoOwner("dir/manifest.in");
+ Artifact outputManifest = getBinArtifactWithNoOwner("dir/MANIFEST");
+ MoreAsserts.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ new SymlinkTreeAction(
+ ActionsTestUtil.NULL_ACTION_OWNER,
+ inputManifest,
+ /*runfiles=*/ null,
+ outputManifest,
+ /*filesetTree=*/ false,
+ createActionEnvironment(false, false),
+ /*enableRunfiles=*/ true,
+ /*inprocessSymlinkCreation=*/ false,
+ /*skipRunfilesManifests=*/ false));
+ }
}
diff --git a/src/test/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategyTest.java b/src/test/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategyTest.java
index 9bb02cc..c50c7f6 100644
--- a/src/test/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategyTest.java
+++ b/src/test/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategyTest.java
@@ -95,7 +95,8 @@
/*filesetTree=*/ false,
ActionEnvironment.EMPTY,
/*enableRunfiles=*/ true,
- /*inprocessSymlinkCreation=*/ false);
+ /*inprocessSymlinkCreation=*/ false,
+ /*skipRunfilesManifests=*/ false);
action.execute(context);
@@ -140,7 +141,8 @@
/*filesetTree=*/ false,
ActionEnvironment.EMPTY,
/*enableRunfiles=*/ true,
- /*inprocessSymlinkCreation=*/ true);
+ /*inprocessSymlinkCreation=*/ true,
+ /*skipRunfilesManifests*/ false);
action.execute(context);
// Check that the OutputService is not used.
