Let artifacts declare that they use content-based paths.
This is basic infrastructure for supporting --experiment_output_paths=content. This change does not implement any actual uses.
The intention is that we can incrementally opt in content-mapping support, i.e. rules can declare which of their outputs should do content mapping. The first test goal will be to opt in java_library compilation.
"Experimental Content-Based Output Paths" design: https://docs.google.com/document/d/17snvmic26-QdGuwVw55Gl0oOufw9sCVuOAvHqGZJFr4/edit
See https://github.com/bazelbuild/bazel/issues/8339
PiperOrigin-RevId: 252148134
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java
index 0eb1853..df5fca7 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java
@@ -40,4 +40,12 @@
* @return the relative path to the input file.
*/
PathFragment getExecPath();
+
+ /**
+ * Returns if this input's file system path includes a digest of its content. See {@link
+ * com.google.devtools.build.lib.analysis.config.BuildConfiguration#useContentBasedOutputPaths}.
+ */
+ default boolean contentBasedPath() {
+ return false;
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/Artifact.java b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java
index 66abc0b..a067437 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/Artifact.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java
@@ -226,7 +226,15 @@
private final ArtifactRoot root;
private final PathFragment execPath;
- private Artifact(ArtifactRoot root, PathFragment execPath) {
+ /**
+ * Content-based output paths are experimental. Only derived artifacts that are explicitly opted
+ * in by their creating rules should use them and only when {@link
+ * com.google.devtools.build.lib.analysis.config.BuildConfiguration#useContentBasedOutputPaths} is
+ * on.
+ */
+ private final boolean contentBasedPath;
+
+ private Artifact(ArtifactRoot root, PathFragment execPath, boolean contentBasedPath) {
Preconditions.checkNotNull(root);
if (execPath.isEmpty()) {
throw new IllegalArgumentException(
@@ -238,6 +246,7 @@
this.hashCode = execPath.hashCode();
this.root = root;
this.execPath = execPath;
+ this.contentBasedPath = contentBasedPath;
}
/** An artifact corresponding to a file in the output tree, generated by an {@link Action}. */
@@ -256,9 +265,19 @@
*/
private Object owner;
- @VisibleForTesting
+ /** Standard constructor for derived artifacts. */
public DerivedArtifact(ArtifactRoot root, PathFragment execPath, ActionLookupKey owner) {
- super(root, execPath);
+ this(root, execPath, owner, /*contentBasedPath=*/ false);
+ }
+
+ /**
+ * Same as {@link #DerivedArtifact(ArtifactRoot, PathFragment, ActionLookupKey)} but includes
+ * tge option to use a content-based path for this artifact (see {@link
+ * com.google.devtools.build.lib.analysis.config.BuildConfiguration#useContentBasedOutputPaths}).
+ */
+ public DerivedArtifact(
+ ArtifactRoot root, PathFragment execPath, ActionLookupKey owner, boolean contentBasedPath) {
+ super(root, execPath, contentBasedPath);
Preconditions.checkState(
!root.getExecPath().isEmpty(), "Derived root has no exec path: %s, %s", root, execPath);
this.rootRelativePath = execPath.relativeTo(root.getExecPath());
@@ -357,7 +376,8 @@
new DerivedArtifact(
root,
rootExecPath.getRelative(rootRelativePath),
- generatingActionKey.getActionLookupKey());
+ generatingActionKey.getActionLookupKey(),
+ /*contentBasedPath=*/ false);
artifact.setGeneratingActionKey(generatingActionKey);
return INTERNER.intern(artifact);
}
@@ -456,6 +476,11 @@
return execPath;
}
+ @Override
+ public boolean contentBasedPath() {
+ return contentBasedPath;
+ }
+
/**
* Returns the path of this Artifact relative to this containing Artifact. Since
* ordinary Artifacts correspond to only one Artifact -- itself -- for ordinary Artifacts,
@@ -536,7 +561,7 @@
@VisibleForTesting
public SourceArtifact(ArtifactRoot root, PathFragment execPath, ArtifactOwner owner) {
- super(root, execPath);
+ super(root, execPath, /*contentBasedPath=*/ false);
this.owner = owner;
}
@@ -596,7 +621,7 @@
@VisibleForTesting
public SpecialArtifact(
ArtifactRoot root, PathFragment execPath, ActionLookupKey owner, SpecialArtifactType type) {
- super(root, execPath, owner);
+ super(root, execPath, owner, /*contentBasedPath=*/ false);
this.type = type;
}
@@ -690,7 +715,8 @@
super(
parentTreeArtifact.getRoot(),
parentTreeArtifact.getExecPath().getRelative(parentRelativePath),
- owner);
+ owner,
+ /*contentBasedPath=*/ false);
Preconditions.checkArgument(
parentTreeArtifact.isTreeArtifact(),
"The parent of TreeFileArtifact (parent-relative path: %s) is not a TreeArtifact: %s",
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java
index c565492..dcc4ccd 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java
@@ -186,7 +186,8 @@
sourceArtifactRoots),
execPath,
owner,
- null);
+ null,
+ /*contentBasedPath=*/ false);
}
@Override
@@ -227,9 +228,23 @@
// TODO(bazel-team): Don't allow root == execRootParent.
public Artifact.DerivedArtifact getDerivedArtifact(
PathFragment rootRelativePath, ArtifactRoot root, ArtifactOwner owner) {
+ return getDerivedArtifact(rootRelativePath, root, owner, /*contentBasedPath=*/ false);
+ }
+
+ /**
+ * Same as {@link #getDerivedArtifact(PathFragment, ArtifactRoot, ArtifactOwner)} but includes the
+ * option to use a content-based path for this artifact (see {@link
+ * com.google.devtools.build.lib.analysis.config.BuildConfiguration#useContentBasedOutputPaths}).
+ */
+ public Artifact.DerivedArtifact getDerivedArtifact(
+ PathFragment rootRelativePath,
+ ArtifactRoot root,
+ ArtifactOwner owner,
+ boolean contentBasedPath) {
validatePath(rootRelativePath, root);
return (Artifact.DerivedArtifact)
- getArtifact(root, root.getExecPath().getRelative(rootRelativePath), owner, null);
+ getArtifact(
+ root, root.getExecPath().getRelative(rootRelativePath), owner, null, contentBasedPath);
}
/**
@@ -248,7 +263,8 @@
root,
root.getExecPath().getRelative(rootRelativePath),
owner,
- SpecialArtifactType.FILESET);
+ SpecialArtifactType.FILESET,
+ /*contentBasedPath=*/ false);
}
/**
@@ -266,7 +282,8 @@
root,
root.getExecPath().getRelative(rootRelativePath),
owner,
- SpecialArtifactType.TREE);
+ SpecialArtifactType.TREE,
+ /*contentBasedPath=*/ false);
}
public Artifact.DerivedArtifact getConstantMetadataArtifact(
@@ -277,7 +294,8 @@
root,
root.getExecPath().getRelative(rootRelativePath),
owner,
- SpecialArtifactType.CONSTANT_METADATA);
+ SpecialArtifactType.CONSTANT_METADATA,
+ /*contentBasedPath=*/ false);
}
/**
@@ -288,12 +306,13 @@
ArtifactRoot root,
PathFragment execPath,
ArtifactOwner owner,
- @Nullable SpecialArtifactType type) {
+ @Nullable SpecialArtifactType type,
+ boolean contentBasedPath) {
Preconditions.checkNotNull(root);
Preconditions.checkNotNull(execPath);
if (!root.isSourceRoot()) {
- return createArtifact(root, execPath, owner, type);
+ return createArtifact(root, execPath, owner, type, contentBasedPath);
}
// Double-checked locking to avoid locking cost when possible.
@@ -307,7 +326,9 @@
// There really should be a safety net that makes it impossible to create two Artifacts
// with the same exec path but a different Owner, but we also need to reuse Artifacts from
// previous builds.
- artifact = (SourceArtifact) createArtifact(root, execPath, owner, type);
+ artifact =
+ (SourceArtifact)
+ createArtifact(root, execPath, owner, type, /*contentBasedPath=*/ false);
sourceArtifactCache.putArtifact(execPath, artifact);
}
} finally {
@@ -321,12 +342,13 @@
ArtifactRoot root,
PathFragment execPath,
ArtifactOwner owner,
- @Nullable SpecialArtifactType type) {
+ @Nullable SpecialArtifactType type,
+ boolean contentBasedPath) {
Preconditions.checkNotNull(owner);
if (type == null) {
return root.isSourceRoot()
? new Artifact.SourceArtifact(root, execPath, owner)
- : new Artifact.DerivedArtifact(root, execPath, (ActionLookupKey) owner);
+ : new Artifact.DerivedArtifact(root, execPath, (ActionLookupKey) owner, contentBasedPath);
} else {
return new Artifact.SpecialArtifact(root, execPath, (ActionLookupKey) owner, type);
}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java
index 2f69670..a72f443 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java
@@ -67,6 +67,14 @@
Artifact.DerivedArtifact getDerivedArtifact(PathFragment rootRelativePath, ArtifactRoot root);
/**
+ * Same as {@link #getDerivedArtifact(PathFragment, ArtifactRoot)} but includes the option to use
+ * a content-based path for this artifact (see {@link
+ * BuildConfiguration#useContentBasedOutputPaths()}).
+ */
+ Artifact.DerivedArtifact getDerivedArtifact(
+ PathFragment rootRelativePath, ArtifactRoot root, boolean contentBasedPath);
+
+ /**
* Returns an artifact for the derived file {@code rootRelativePath} whose changes do not cause a
* rebuild.
*
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java
index b31208d..f6e6f76 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java
@@ -282,9 +282,15 @@
@Override
public Artifact.DerivedArtifact getDerivedArtifact(
PathFragment rootRelativePath, ArtifactRoot root) {
+ return getDerivedArtifact(rootRelativePath, root, /*contentBasedPath=*/ false);
+ }
+
+ @Override
+ public Artifact.DerivedArtifact getDerivedArtifact(
+ PathFragment rootRelativePath, ArtifactRoot root, boolean contentBasedPath) {
Preconditions.checkState(enabled);
return dedupAndTrackArtifactAndOrigin(
- artifactFactory.getDerivedArtifact(rootRelativePath, root, getOwner()),
+ artifactFactory.getDerivedArtifact(rootRelativePath, root, getOwner(), contentBasedPath),
extendedSanityChecks ? new Throwable() : null);
}
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java
index b4ab0d7..72549d1 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java
@@ -686,7 +686,17 @@
@Override
public Artifact.DerivedArtifact getPackageRelativeArtifact(
PathFragment relative, ArtifactRoot root) {
- return getDerivedArtifact(getPackageDirectory().getRelative(relative), root);
+ return getPackageRelativeArtifact(relative, root, /*contentBasedPath=*/ false);
+ }
+
+ /**
+ * Same as {@link #getPackageRelativeArtifact(PathFragment, ArtifactRoot)} but includes the option
+ * option to use a content-based path for this artifact (see {@link
+ * BuildConfiguration#useContentBasedOutputPaths()}).
+ */
+ private Artifact.DerivedArtifact getPackageRelativeArtifact(
+ PathFragment relative, ArtifactRoot root, boolean contentBasedPath) {
+ return getDerivedArtifact(getPackageDirectory().getRelative(relative), root, contentBasedPath);
}
/**
@@ -694,7 +704,17 @@
* guaranteeing that it never clashes with artifacts created by rules in other packages.
*/
public Artifact getPackageRelativeArtifact(String relative, ArtifactRoot root) {
- return getPackageRelativeArtifact(PathFragment.create(relative), root);
+ return getPackageRelativeArtifact(relative, root, /*contentBasedPath=*/ false);
+ }
+
+ /**
+ * Same as {@link #getPackageRelativeArtifact(String, ArtifactRoot)} but includes the option to
+ * use a content-based path for this artifact (see {@link
+ * BuildConfiguration#useContentBasedOutputPaths()}).
+ */
+ private Artifact getPackageRelativeArtifact(
+ String relative, ArtifactRoot root, boolean contentBasedPath) {
+ return getPackageRelativeArtifact(PathFragment.create(relative), root, contentBasedPath);
}
@Override
@@ -712,10 +732,20 @@
@Override
public Artifact.DerivedArtifact getDerivedArtifact(
PathFragment rootRelativePath, ArtifactRoot root) {
+ return getDerivedArtifact(rootRelativePath, root, /*contentBasedPath=*/ false);
+ }
+
+ /**
+ * Same as {@link #getDerivedArtifact(PathFragment, ArtifactRoot)} but includes the option to use
+ * a content-based path for this artifact (see {@link
+ * BuildConfiguration#useContentBasedOutputPaths()}).
+ */
+ public Artifact.DerivedArtifact getDerivedArtifact(
+ PathFragment rootRelativePath, ArtifactRoot root, boolean contentBasedPath) {
Preconditions.checkState(rootRelativePath.startsWith(getPackageDirectory()),
"Output artifact '%s' not under package directory '%s' for target '%s'",
rootRelativePath, getPackageDirectory(), getLabel());
- return getAnalysisEnvironment().getDerivedArtifact(rootRelativePath, root);
+ return getAnalysisEnvironment().getDerivedArtifact(rootRelativePath, root, contentBasedPath);
}
@Override
@@ -1351,6 +1381,16 @@
@Override
public Artifact getImplicitOutputArtifact(ImplicitOutputsFunction function)
throws InterruptedException {
+ return getImplicitOutputArtifact(function, /*contentBasedPath=*/ false);
+ }
+
+ /**
+ * Same as {@link #getImplicitOutputArtifact(ImplicitOutputsFunction)} but includes the option to
+ * use a content-based path for this artifact (see {@link
+ * BuildConfiguration#useContentBasedOutputPaths()}).
+ */
+ public Artifact getImplicitOutputArtifact(
+ ImplicitOutputsFunction function, boolean contentBasedPath) throws InterruptedException {
Iterable<String> result;
try {
result =
@@ -1360,14 +1400,23 @@
// It's ok as long as we don't use this method from Skylark.
throw new IllegalStateException(e);
}
- return getImplicitOutputArtifact(Iterables.getOnlyElement(result));
+ return getImplicitOutputArtifact(Iterables.getOnlyElement(result), contentBasedPath);
}
/**
* Only use from Skylark. Returns the implicit output artifact for a given output path.
*/
public Artifact getImplicitOutputArtifact(String path) {
- return getPackageRelativeArtifact(path, getBinOrGenfilesDirectory());
+ return getImplicitOutputArtifact(path, /*contentBasedPath=*/ false);
+ }
+
+ /**
+ * Same as {@link #getImplicitOutputArtifact(String)} but includes the option to use a a
+ * content-based path for this artifact (see {@link
+ * BuildConfiguration#useContentBasedOutputPaths()}).
+ */
+ private Artifact getImplicitOutputArtifact(String path, boolean contentBasedPath) {
+ return getPackageRelativeArtifact(path, getBinOrGenfilesDirectory(), contentBasedPath);
}
/**
diff --git a/src/test/java/com/google/devtools/build/lib/actions/ArtifactTest.java b/src/test/java/com/google/devtools/build/lib/actions/ArtifactTest.java
index 4dd23eb..6e05f61 100644
--- a/src/test/java/com/google/devtools/build/lib/actions/ArtifactTest.java
+++ b/src/test/java/com/google/devtools/build/lib/actions/ArtifactTest.java
@@ -491,4 +491,18 @@
ArtifactRoot.asSourceRoot(Root.fromPath(scratch.dir("/"))),
scratch.file("/aaa/bbb/ccc/ddd"));
}
+
+ @Test
+ public void canDeclareContentBasedOutput() throws Exception {
+ Path execRoot = scratch.getFileSystem().getPath("/");
+ ArtifactRoot root = ArtifactRoot.asDerivedRoot(execRoot, scratch.dir("/newRoot"));
+ assertThat(
+ new Artifact.DerivedArtifact(
+ root,
+ PathFragment.create("newRoot/my.output"),
+ ActionsTestUtil.NULL_ARTIFACT_OWNER,
+ /*contentBasedPath=*/ true)
+ .contentBasedPath())
+ .isTrue();
+ }
}
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestUtil.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestUtil.java
index 09caee3..7b5eb43 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestUtil.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestUtil.java
@@ -126,6 +126,12 @@
}
@Override
+ public Artifact.DerivedArtifact getDerivedArtifact(
+ PathFragment rootRelativePath, ArtifactRoot root, boolean contentBasedPath) {
+ return original.getDerivedArtifact(rootRelativePath, root, contentBasedPath);
+ }
+
+ @Override
public Artifact getConstantMetadataArtifact(PathFragment rootRelativePath, ArtifactRoot root) {
return original.getConstantMetadataArtifact(rootRelativePath, root);
}
@@ -371,6 +377,12 @@
}
@Override
+ public Artifact.DerivedArtifact getDerivedArtifact(
+ PathFragment rootRelativePath, ArtifactRoot root, boolean contentBasedPath) {
+ return null;
+ }
+
+ @Override
public Artifact getStableWorkspaceStatusArtifact() {
return null;
}
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
index 32824a5..834d8ff 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
@@ -2011,6 +2011,12 @@
}
@Override
+ public Artifact.DerivedArtifact getDerivedArtifact(
+ PathFragment rootRelativePath, ArtifactRoot root, boolean contentBasedPath) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public Artifact getStableWorkspaceStatusArtifact() {
throw new UnsupportedOperationException();
}
diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionTest.java b/src/test/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionTest.java
index d4c6d21..86289c3 100644
--- a/src/test/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionTest.java
@@ -18,6 +18,7 @@
import static com.google.devtools.build.lib.testutil.MoreAsserts.assertThrows;
import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
@@ -95,6 +96,14 @@
return CppLinkActionTest.this.getDerivedArtifact(
rootRelativePath, root, ActionsTestUtil.NULL_ARTIFACT_OWNER);
}
+
+ @Override
+ public Artifact.DerivedArtifact getDerivedArtifact(
+ PathFragment rootRelativePath, ArtifactRoot root, boolean contentBasedPaths) {
+ Preconditions.checkArgument(
+ !contentBasedPaths, "C++ tests don't use content-based outputs");
+ return getDerivedArtifact(rootRelativePath, root);
+ }
},
masterConfig);
}
