Implement MetadataProvider in RemoteActionFileSystem.
This makes it possible (in a followup change) to delete the downloadFile()
prefetcher method in favor of prefetchFiles().
When the action filesystem implements MetadataProvider, Skyframe considers it the authoritative source for input metadata [1]. Therefore, getInput() and getMetadata() need to look in all of the right places (the action input map, the output mapping, the remote output tree, and the local filesystem) and get all of the combinations of source/output and local/remote right.
[1] https://cs.opensource.google/bazel/bazel/+/master:src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java;l=859;drc=9f8395661b2a74299cf01b1a1cdd04cc4f25dcf0
PiperOrigin-RevId: 515642065
Change-Id: I62caf3aafa4acc23f70cc4511997335e82ebb12a
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionFileSystem.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionFileSystem.java
index 72fb436..2998356 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionFileSystem.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionFileSystem.java
@@ -31,6 +31,7 @@
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
+import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.actions.RemoteFileStatus;
import com.google.devtools.build.lib.actions.cache.MetadataInjector;
import com.google.devtools.build.lib.clock.Clock;
@@ -68,10 +69,11 @@
*
* <p>This implementation only supports creating local action outputs.
*/
-public class RemoteActionFileSystem extends DelegateFileSystem {
+public class RemoteActionFileSystem extends DelegateFileSystem implements MetadataProvider {
private final PathFragment execRoot;
private final PathFragment outputBase;
+ private final MetadataProvider fileCache;
private final ActionInputMap inputArtifactData;
private final ImmutableMap<PathFragment, Artifact> outputMapping;
private final RemoteActionInputFetcher inputFetcher;
@@ -85,6 +87,7 @@
String relativeOutputPath,
ActionInputMap inputArtifactData,
Iterable<Artifact> outputArtifacts,
+ MetadataProvider fileCache,
RemoteActionInputFetcher inputFetcher) {
super(localDelegate);
this.execRoot = checkNotNull(execRootFragment, "execRootFragment");
@@ -92,6 +95,7 @@
this.inputArtifactData = checkNotNull(inputArtifactData, "inputArtifactData");
this.outputMapping =
stream(outputArtifacts).collect(toImmutableMap(Artifact::getExecPath, a -> a));
+ this.fileCache = checkNotNull(fileCache, "fileCache");
this.inputFetcher = checkNotNull(inputFetcher, "inputFetcher");
this.remoteOutputTree = new RemoteInMemoryFileSystem(getDigestFunction());
}
@@ -108,7 +112,11 @@
/** Returns true if {@code path} is a file that's stored remotely. */
boolean isRemote(Path path) {
- return getRemoteMetadata(path.asFragment()) != null;
+ return isRemote(path.asFragment());
+ }
+
+ private boolean isRemote(PathFragment path) {
+ return getRemoteMetadata(path) != null;
}
public void updateContext(MetadataInjector metadataInjector) {
@@ -240,6 +248,8 @@
@Override
protected InputStream getInputStream(PathFragment path) throws IOException {
downloadFileIfRemote(path);
+ // TODO(tjgq): Consider only falling back to the local filesystem for source (non-output) files.
+ // See getMetadata() for why this isn't currently possible.
return super.getInputStream(path);
}
@@ -539,25 +549,54 @@
};
}
+ @Override
@Nullable
- protected ActionInput getActionInput(PathFragment path) {
- PathFragment execPath = path.relativeTo(execRoot);
- return inputArtifactData.getInput(execPath.getPathString());
+ public ActionInput getInput(String execPath) {
+ ActionInput input = inputArtifactData.getInput(execPath);
+ if (input != null) {
+ return input;
+ }
+ input = outputMapping.get(PathFragment.create(execPath));
+ if (input != null) {
+ return input;
+ }
+ if (!isOutput(execRoot.getRelative(execPath))) {
+ return fileCache.getInput(execPath);
+ }
+ return null;
+ }
+
+ ActionInput getActionInput(PathFragment path) {
+ return getInput(path.relativeTo(execRoot).getPathString());
}
@Nullable
- protected RemoteFileArtifactValue getRemoteMetadata(PathFragment path) {
- if (!isOutput(path)) {
- return null;
+ @Override
+ public FileArtifactValue getMetadata(ActionInput input) throws IOException {
+ PathFragment execPath = input.getExecPath();
+ FileArtifactValue m = getMetadataByExecPath(execPath);
+ if (m != null) {
+ return m;
}
- PathFragment execPath = path.relativeTo(execRoot);
+ // TODO(tjgq): Consider only falling back to the local filesystem for source (non-output) files.
+ // The output fallback is needed when an undeclared output of a spawn is consumed by another
+ // spawn within the same action; specifically, when the first spawn is local but the second is
+ // remote, or, in the context of a failed test attempt, when both spawns are remote but the
+ // first one fails. In both cases, we don't currently inject the output metadata for the first
+ // spawn; if we did so, then we could stop falling back here.
+ return fileCache.getMetadata(input);
+ }
+
+ @Nullable
+ private FileArtifactValue getMetadataByExecPath(PathFragment execPath) {
FileArtifactValue m = inputArtifactData.getMetadata(execPath);
- if (m != null && m.isRemote()) {
- return (RemoteFileArtifactValue) m;
+ if (m != null) {
+ return m;
}
RemoteFileInfo remoteFile =
- remoteOutputTree.getRemoteFileInfo(path, /* followSymlinks= */ true);
+ remoteOutputTree.getRemoteFileInfo(
+ execRoot.getRelative(execPath), /* followSymlinks= */ true);
if (remoteFile != null) {
return createRemoteMetadata(remoteFile);
}
@@ -566,12 +605,23 @@
}
@Nullable
+ RemoteFileArtifactValue getRemoteMetadata(PathFragment path) {
+ if (!isOutput(path)) {
+ return null;
+ }
+ FileArtifactValue m = getMetadataByExecPath(path.relativeTo(execRoot));
+ if (m != null && m.isRemote()) {
+ return (RemoteFileArtifactValue) m;
+ }
+ return null;
+ }
+
+ @Nullable
private TreeArtifactValue getRemoteTreeMetadata(PathFragment path) {
if (!isOutput(path)) {
return null;
}
- PathFragment execPath = path.relativeTo(execRoot);
- TreeArtifactValue m = inputArtifactData.getTreeMetadata(execPath);
+ TreeArtifactValue m = inputArtifactData.getTreeMetadata(path.relativeTo(execRoot));
// TODO: Handle partially remote tree artifacts.
if (m != null && m.isEntirelyRemote()) {
return m;
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
index ef57db3..85323a0 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -1030,6 +1030,7 @@
remoteOutputService.setActionInputFetcher(actionInputFetcher);
remoteOutputService.setLeaseService(leaseService);
+ remoteOutputService.setFileCacheSupplier(env::getFileCache);
env.getEventBus().register(remoteOutputService);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java
index 2ea7a0f..d294cb0 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java
@@ -24,6 +24,7 @@
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
+import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.actions.cache.MetadataHandler;
import com.google.devtools.build.lib.actions.cache.MetadataInjector;
import com.google.devtools.build.lib.buildtool.buildevent.ExecutionPhaseCompleteEvent;
@@ -39,6 +40,7 @@
import java.io.IOException;
import java.util.Map;
import java.util.UUID;
+import java.util.function.Supplier;
import javax.annotation.Nullable;
/** Output service implementation for the remote module */
@@ -46,6 +48,7 @@
@Nullable private RemoteActionInputFetcher actionInputFetcher;
@Nullable private LeaseService leaseService;
+ @Nullable private Supplier<MetadataProvider> fileCacheSupplier;
void setActionInputFetcher(RemoteActionInputFetcher actionInputFetcher) {
this.actionInputFetcher = Preconditions.checkNotNull(actionInputFetcher, "actionInputFetcher");
@@ -55,6 +58,10 @@
this.leaseService = leaseService;
}
+ void setFileCacheSupplier(Supplier<MetadataProvider> fileCacheSupplier) {
+ this.fileCacheSupplier = fileCacheSupplier;
+ }
+
@Override
public ActionFileSystemType actionFileSystemType() {
return actionInputFetcher != null
@@ -79,6 +86,7 @@
relativeOutputPath,
inputArtifactData,
outputArtifacts,
+ fileCacheSupplier.get(),
actionInputFetcher);
}
@@ -177,6 +185,7 @@
relativeOutputPath,
actionInputMap,
ImmutableList.of(),
+ fileCacheSupplier.get(),
actionInputFetcher);
return ArtifactPathResolver.createPathResolver(remoteFileSystem, fileSystem.getPath(execRoot));
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/StaticMetadataProvider.java b/src/main/java/com/google/devtools/build/lib/remote/util/StaticMetadataProvider.java
index 1d12166..2d1a194 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/util/StaticMetadataProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/util/StaticMetadataProvider.java
@@ -17,27 +17,46 @@
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.MetadataProvider;
+import java.util.Collection;
import java.util.Map;
import javax.annotation.Nullable;
/** A {@link MetadataProvider} backed by static data */
public final class StaticMetadataProvider implements MetadataProvider {
- private final ImmutableMap<ActionInput, FileArtifactValue> metadata;
+ private static final StaticMetadataProvider EMPTY = new StaticMetadataProvider(ImmutableMap.of());
- public StaticMetadataProvider(Map<ActionInput, FileArtifactValue> metadata) {
- this.metadata = ImmutableMap.copyOf(metadata);
+ public static StaticMetadataProvider empty() {
+ return EMPTY;
+ }
+
+ private final ImmutableMap<ActionInput, FileArtifactValue> inputToMetadata;
+ private final ImmutableMap<String, ActionInput> execPathToInput;
+
+ public StaticMetadataProvider(Map<ActionInput, FileArtifactValue> inputToMetadata) {
+ this.inputToMetadata = ImmutableMap.copyOf(inputToMetadata);
+ this.execPathToInput = constructExecPathToInputMap(inputToMetadata.keySet());
+ }
+
+ private static ImmutableMap<String, ActionInput> constructExecPathToInputMap(
+ Collection<ActionInput> inputs) {
+ ImmutableMap.Builder<String, ActionInput> builder =
+ ImmutableMap.builderWithExpectedSize(inputs.size());
+ for (ActionInput input : inputs) {
+ builder.put(input.getExecPath().getPathString(), input);
+ }
+ return builder.buildOrThrow();
}
@Nullable
@Override
public FileArtifactValue getMetadata(ActionInput input) {
- return metadata.get(input);
+ return inputToMetadata.get(input);
}
@Nullable
@Override
public ActionInput getInput(String execPath) {
- throw new UnsupportedOperationException();
+ return execPathToInput.get(execPath);
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java
index be4d348..0233d79 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamBuildEventArtifactUploaderTest.java
@@ -59,6 +59,7 @@
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.remote.util.RxNoGlobalErrorsRule;
+import com.google.devtools.build.lib.remote.util.StaticMetadataProvider;
import com.google.devtools.build.lib.remote.util.TestUtils;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.FileSystem;
@@ -365,6 +366,7 @@
outputRoot.getRoot().asPath().relativeTo(execRoot).getPathString(),
outputs,
ImmutableList.of(artifact),
+ StaticMetadataProvider.empty(),
actionInputFetcher);
Path remotePath = remoteFs.getPath(artifact.getPath().getPathString());
assertThat(remotePath.getFileSystem()).isEqualTo(remoteFs);
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTest.java
index 9e1eb36..5be2fd8 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTest.java
@@ -13,6 +13,7 @@
// limitations under the License.
package com.google.devtools.build.lib.remote;
+import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static org.mockito.ArgumentMatchers.any;
@@ -35,14 +36,17 @@
import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
+import com.google.devtools.build.lib.actions.MetadataProvider;
import com.google.devtools.build.lib.actions.cache.MetadataInjector;
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
+import com.google.devtools.build.lib.remote.util.StaticMetadataProvider;
import com.google.devtools.build.lib.skyframe.TreeArtifactValue;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.Symlinks;
import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
@@ -61,12 +65,15 @@
private static final DigestHashFunction HASH_FUNCTION = DigestHashFunction.SHA256;
+ private static final String RELATIVE_OUTPUT_PATH = "out";
+
private final RemoteActionInputFetcher inputFetcher = mock(RemoteActionInputFetcher.class);
private final MetadataInjector metadataInjector = mock(MetadataInjector.class);
private final FileSystem fs = new InMemoryFileSystem(HASH_FUNCTION);
private final Path execRoot = fs.getPath("/exec");
+ private final ArtifactRoot sourceRoot = ArtifactRoot.asSourceRoot(Root.fromPath(execRoot));
private final ArtifactRoot outputRoot =
- ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, "out");
+ ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, RELATIVE_OUTPUT_PATH);
@Before
public void setUp() throws IOException {
@@ -74,15 +81,17 @@
}
@Override
- protected RemoteActionFileSystem createActionFileSystem(
- ActionInputMap inputs, Iterable<Artifact> outputs) throws IOException {
+ protected FileSystem createActionFileSystem(
+ ActionInputMap inputs, Iterable<Artifact> outputs, MetadataProvider fileCache)
+ throws IOException {
RemoteActionFileSystem remoteActionFileSystem =
new RemoteActionFileSystem(
fs,
execRoot.asFragment(),
- outputRoot.getRoot().asPath().relativeTo(execRoot).getPathString(),
+ RELATIVE_OUTPUT_PATH,
inputs,
outputs,
+ fileCache,
inputFetcher);
remoteActionFileSystem.updateContext(metadataInjector);
remoteActionFileSystem.createDirectoryAndParents(outputRoot.getRoot().asPath().asFragment());
@@ -138,13 +147,160 @@
}
@Test
+ public void getInput_fromInputArtifactData_forLocalArtifact() throws Exception {
+ ActionInputMap inputs = new ActionInputMap(1);
+ Artifact artifact = createLocalArtifact("local-file", "local contents", inputs);
+ MetadataProvider actionFs = (MetadataProvider) createActionFileSystem(inputs);
+
+ assertThat(actionFs.getInput(artifact.getExecPathString())).isEqualTo(artifact);
+ }
+
+ @Test
+ public void getInput_fromInputArtifactData_forRemoteArtifact() throws Exception {
+ ActionInputMap inputs = new ActionInputMap(1);
+ Artifact artifact = createRemoteArtifact("remote-file", "remote contents", inputs);
+ MetadataProvider actionFs = (MetadataProvider) createActionFileSystem(inputs);
+
+ assertThat(actionFs.getInput(artifact.getExecPathString())).isEqualTo(artifact);
+ }
+
+ @Test
+ public void getInput_fromOutputMapping() throws Exception {
+ Artifact artifact = ActionsTestUtil.createArtifact(outputRoot, "out");
+ MetadataProvider actionFs =
+ (MetadataProvider)
+ createActionFileSystem(new ActionInputMap(0), ImmutableList.of(artifact));
+
+ assertThat(actionFs.getInput(artifact.getExecPathString())).isEqualTo(artifact);
+ }
+
+ @Test
+ public void getInput_fromFileCache_forSourceFile() throws Exception {
+ Artifact artifact = ActionsTestUtil.createArtifact(sourceRoot, "src");
+ FileArtifactValue metadata =
+ FileArtifactValue.createForNormalFile(new byte[] {1, 2, 3}, /* proxy= */ null, 42);
+ MetadataProvider actionFs =
+ (MetadataProvider)
+ createActionFileSystem(
+ new ActionInputMap(0),
+ ImmutableList.of(),
+ new StaticMetadataProvider(ImmutableMap.of(artifact, metadata)));
+
+ assertThat(actionFs.getInput(artifact.getExecPathString())).isEqualTo(artifact);
+ }
+
+ @Test
+ public void getInput_fromFileCache_notForOutputFile() throws Exception {
+ Artifact artifact = ActionsTestUtil.createArtifact(outputRoot, "out");
+ FileArtifactValue metadata =
+ FileArtifactValue.createForNormalFile(new byte[] {1, 2, 3}, /* proxy= */ null, 42);
+ MetadataProvider actionFs =
+ (MetadataProvider)
+ createActionFileSystem(
+ new ActionInputMap(0),
+ ImmutableList.of(),
+ new StaticMetadataProvider(ImmutableMap.of(artifact, metadata)));
+
+ assertThat(actionFs.getInput(artifact.getExecPathString())).isNull();
+ }
+
+ @Test
+ public void getInput_notFound() throws Exception {
+ MetadataProvider actionFs = (MetadataProvider) createActionFileSystem();
+
+ assertThat(actionFs.getInput("some-path")).isNull();
+ }
+
+ @Test
+ public void getMetadata_fromInputArtifactData_forLocalArtifact() throws Exception {
+ ActionInputMap inputs = new ActionInputMap(1);
+ Artifact artifact = createLocalArtifact("local-file", "local contents", inputs);
+ FileArtifactValue metadata = checkNotNull(inputs.getMetadata(artifact));
+ MetadataProvider actionFs = (MetadataProvider) createActionFileSystem(inputs);
+
+ assertThat(actionFs.getMetadata(artifact)).isEqualTo(metadata);
+ }
+
+ @Test
+ public void getMetadata_fromInputArtifactData_forRemoteArtifact() throws Exception {
+ ActionInputMap inputs = new ActionInputMap(1);
+ Artifact artifact = createRemoteArtifact("remote-file", "remote contents", inputs);
+ FileArtifactValue metadata = checkNotNull(inputs.getMetadata(artifact));
+ MetadataProvider actionFs = (MetadataProvider) createActionFileSystem(inputs);
+
+ assertThat(actionFs.getMetadata(artifact)).isEqualTo(metadata);
+ }
+
+ @Test
+ public void getMetadata_fromRemoteOutputTree_forDeclaredOutput() throws Exception {
+ Artifact artifact = ActionsTestUtil.createArtifact(outputRoot, "out");
+ MetadataProvider actionFs =
+ (MetadataProvider)
+ createActionFileSystem(new ActionInputMap(0), ImmutableList.of(artifact));
+
+ FileArtifactValue metadata =
+ injectRemoteFile((FileSystem) actionFs, artifact.getPath().asFragment(), "content");
+
+ assertThat(actionFs.getMetadata(artifact)).isEqualTo(metadata);
+ }
+
+ @Test
+ public void getMetadata_fromRemoteOutputTree_forUndeclaredOutput() throws Exception {
+ Artifact artifact = ActionsTestUtil.createArtifact(outputRoot, "out");
+ MetadataProvider actionFs = (MetadataProvider) createActionFileSystem();
+
+ FileArtifactValue metadata =
+ injectRemoteFile((FileSystem) actionFs, artifact.getPath().asFragment(), "content");
+
+ assertThat(actionFs.getMetadata(artifact)).isEqualTo(metadata);
+ }
+
+ @Test
+ public void getMetadata_fromFileCache_forSourceFile() throws Exception {
+ Artifact artifact = ActionsTestUtil.createArtifact(sourceRoot, "src");
+ FileArtifactValue metadata =
+ FileArtifactValue.createForNormalFile(new byte[] {1, 2, 3}, /* proxy= */ null, 42);
+ MetadataProvider actionFs =
+ (MetadataProvider)
+ createActionFileSystem(
+ new ActionInputMap(0),
+ ImmutableList.of(),
+ new StaticMetadataProvider(ImmutableMap.of(artifact, metadata)));
+
+ assertThat(actionFs.getMetadata(artifact)).isEqualTo(metadata);
+ }
+
+ @Test
+ public void getMetadata_fromFileCache_forOutputFile() throws Exception {
+ Artifact artifact = ActionsTestUtil.createArtifact(outputRoot, "out");
+ FileArtifactValue metadata =
+ FileArtifactValue.createForNormalFile(new byte[] {1, 2, 3}, /* proxy= */ null, 42);
+ MetadataProvider actionFs =
+ (MetadataProvider)
+ createActionFileSystem(
+ new ActionInputMap(0),
+ ImmutableList.of(),
+ new StaticMetadataProvider(ImmutableMap.of(artifact, metadata)));
+
+ assertThat(actionFs.getMetadata(artifact)).isEqualTo(metadata);
+ }
+
+ @Test
+ public void getMetadata_notFound() throws Exception {
+ Artifact artifact = ActionsTestUtil.createArtifact(outputRoot, "out");
+ MetadataProvider actionFs = (MetadataProvider) createActionFileSystem();
+
+ assertThat(actionFs.getMetadata(artifact)).isNull();
+ }
+
+ @Test
public void createSymbolicLink_localFileArtifact() throws IOException {
// arrange
ActionInputMap inputs = new ActionInputMap(1);
Artifact localArtifact = createLocalArtifact("local-file", "local contents", inputs);
Artifact outputArtifact = ActionsTestUtil.createArtifact(outputRoot, "out");
ImmutableList<Artifact> outputs = ImmutableList.of(outputArtifact);
- RemoteActionFileSystem actionFs = createActionFileSystem(inputs, outputs);
+ FileSystem actionFs = createActionFileSystem(inputs, outputs);
// act
PathFragment linkPath = outputArtifact.getPath().asFragment();
@@ -155,13 +311,13 @@
// assert
assertThat(symlinkActionFs.getFileSystem()).isSameInstanceAs(actionFs);
assertThat(symlinkActionFs.readSymbolicLink()).isEqualTo(targetPath);
- assertThat(actionFs.getLocalFileSystem().getPath(linkPath).readSymbolicLink())
+ assertThat(getLocalFileSystem(actionFs).getPath(linkPath).readSymbolicLink())
.isEqualTo(targetPath);
- assertThat(actionFs.getLocalFileSystem().getPath(linkPath).readSymbolicLink())
+ assertThat(getLocalFileSystem(actionFs).getPath(linkPath).readSymbolicLink())
.isEqualTo(targetPath);
// act
- actionFs.flush();
+ ((RemoteActionFileSystem) actionFs).flush();
// assert
verifyNoInteractions(metadataInjector);
@@ -174,7 +330,7 @@
Artifact remoteArtifact = createRemoteArtifact("remote-file", "remote contents", inputs);
Artifact outputArtifact = ActionsTestUtil.createArtifact(outputRoot, "out");
ImmutableList<Artifact> outputs = ImmutableList.of(outputArtifact);
- RemoteActionFileSystem actionFs = createActionFileSystem(inputs, outputs);
+ FileSystem actionFs = createActionFileSystem(inputs, outputs);
// act
PathFragment linkPath = outputArtifact.getPath().asFragment();
@@ -186,13 +342,13 @@
assertThat(symlinkActionFs.getFileSystem()).isSameInstanceAs(actionFs);
assertThat(symlinkActionFs.readSymbolicLink()).isEqualTo(targetPath);
assertThat(outputArtifact.getPath().readSymbolicLink()).isEqualTo(targetPath);
- assertThat(actionFs.getLocalFileSystem().getPath(linkPath).readSymbolicLink())
+ assertThat(getLocalFileSystem(actionFs).getPath(linkPath).readSymbolicLink())
.isEqualTo(targetPath);
- assertThat(actionFs.getLocalFileSystem().getPath(linkPath).readSymbolicLink())
+ assertThat(getLocalFileSystem(actionFs).getPath(linkPath).readSymbolicLink())
.isEqualTo(targetPath);
// act
- actionFs.flush();
+ ((RemoteActionFileSystem) actionFs).flush();
// assert
ArgumentCaptor<FileArtifactValue> metadataCaptor =
@@ -214,7 +370,7 @@
SpecialArtifact outputArtifact =
ActionsTestUtil.createTreeArtifactWithGeneratingAction(outputRoot, "out");
ImmutableList<Artifact> outputs = ImmutableList.of(outputArtifact);
- RemoteActionFileSystem actionFs = createActionFileSystem(inputs, outputs);
+ FileSystem actionFs = createActionFileSystem(inputs, outputs);
// act
PathFragment linkPath = outputArtifact.getPath().asFragment();
@@ -225,13 +381,13 @@
// assert
assertThat(symlinkActionFs.getFileSystem()).isSameInstanceAs(actionFs);
assertThat(symlinkActionFs.readSymbolicLink()).isEqualTo(targetPath);
- assertThat(actionFs.getLocalFileSystem().getPath(linkPath).readSymbolicLink())
+ assertThat(getLocalFileSystem(actionFs).getPath(linkPath).readSymbolicLink())
.isEqualTo(targetPath);
- assertThat(actionFs.getLocalFileSystem().getPath(linkPath).readSymbolicLink())
+ assertThat(getLocalFileSystem(actionFs).getPath(linkPath).readSymbolicLink())
.isEqualTo(targetPath);
// act
- actionFs.flush();
+ ((RemoteActionFileSystem) actionFs).flush();
// assert
verifyNoInteractions(metadataInjector);
@@ -247,7 +403,7 @@
SpecialArtifact outputArtifact =
ActionsTestUtil.createTreeArtifactWithGeneratingAction(outputRoot, "out");
ImmutableList<Artifact> outputs = ImmutableList.of(outputArtifact);
- RemoteActionFileSystem actionFs = createActionFileSystem(inputs, outputs);
+ FileSystem actionFs = createActionFileSystem(inputs, outputs);
// act
PathFragment linkPath = outputArtifact.getPath().asFragment();
@@ -258,13 +414,13 @@
// assert
assertThat(symlinkActionFs.getFileSystem()).isSameInstanceAs(actionFs);
assertThat(symlinkActionFs.readSymbolicLink()).isEqualTo(targetPath);
- assertThat(actionFs.getLocalFileSystem().getPath(linkPath).readSymbolicLink())
+ assertThat(getLocalFileSystem(actionFs).getPath(linkPath).readSymbolicLink())
.isEqualTo(targetPath);
- assertThat(actionFs.getLocalFileSystem().getPath(linkPath).readSymbolicLink())
+ assertThat(getLocalFileSystem(actionFs).getPath(linkPath).readSymbolicLink())
.isEqualTo(targetPath);
// act
- actionFs.flush();
+ ((RemoteActionFileSystem) actionFs).flush();
// assert
ArgumentCaptor<TreeArtifactValue> metadataCaptor =
@@ -282,7 +438,7 @@
SpecialArtifact outputArtifact =
ActionsTestUtil.createUnresolvedSymlinkArtifact(outputRoot, "out");
ImmutableList<Artifact> outputs = ImmutableList.of(outputArtifact);
- RemoteActionFileSystem actionFs = createActionFileSystem(inputs, outputs);
+ FileSystem actionFs = createActionFileSystem(inputs, outputs);
PathFragment targetPath = PathFragment.create("some/path");
// act
@@ -293,13 +449,13 @@
// assert
assertThat(symlinkActionFs.getFileSystem()).isSameInstanceAs(actionFs);
assertThat(symlinkActionFs.readSymbolicLink()).isEqualTo(targetPath);
- assertThat(actionFs.getLocalFileSystem().getPath(linkPath).readSymbolicLink())
+ assertThat(getLocalFileSystem(actionFs).getPath(linkPath).readSymbolicLink())
.isEqualTo(targetPath);
- assertThat(actionFs.getLocalFileSystem().getPath(linkPath).readSymbolicLink())
+ assertThat(getLocalFileSystem(actionFs).getPath(linkPath).readSymbolicLink())
.isEqualTo(targetPath);
// act
- actionFs.flush();
+ ((RemoteActionFileSystem) actionFs).flush();
// assert
verifyNoInteractions(metadataInjector);
@@ -321,13 +477,18 @@
}
@Override
- protected void injectRemoteFile(FileSystem actionFs, PathFragment path, String content)
- throws IOException {
+ protected FileArtifactValue injectRemoteFile(
+ FileSystem actionFs, PathFragment path, String content) throws IOException {
byte[] contentBytes = content.getBytes(StandardCharsets.UTF_8);
HashCode hashCode = HASH_FUNCTION.getHashFunction().hashBytes(contentBytes);
((RemoteActionFileSystem) actionFs)
.injectRemoteFile(
path, hashCode.asBytes(), contentBytes.length, /* expireAtEpochMilli= */ -1);
+ return RemoteFileArtifactValue.create(
+ hashCode.asBytes(),
+ contentBytes.length,
+ /* locationIndex= */ 1,
+ /* expireAtEpochMilli= */ -1);
}
@Override
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTestBase.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTestBase.java
index 638faa1..33c532e 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTestBase.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTestBase.java
@@ -20,28 +20,38 @@
import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.actions.ActionInputMap;
import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.FileArtifactValue;
+import com.google.devtools.build.lib.actions.MetadataProvider;
+import com.google.devtools.build.lib.remote.util.StaticMetadataProvider;
import com.google.devtools.build.lib.vfs.Dirent;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.junit.Test;
public abstract class RemoteActionFileSystemTestBase {
- protected abstract FileSystem createActionFileSystem(
- ActionInputMap inputs, Iterable<Artifact> outputs) throws IOException;
- protected FileSystem createActionFileSystem() throws IOException {
- ActionInputMap inputs = new ActionInputMap(0);
- return createActionFileSystem(inputs);
+ protected abstract FileSystem createActionFileSystem(
+ ActionInputMap inputs, Iterable<Artifact> outputs, MetadataProvider fileCache)
+ throws IOException;
+
+ protected FileSystem createActionFileSystem(ActionInputMap inputs, Iterable<Artifact> outputs)
+ throws IOException {
+ return createActionFileSystem(inputs, outputs, StaticMetadataProvider.empty());
}
protected FileSystem createActionFileSystem(ActionInputMap inputs) throws IOException {
return createActionFileSystem(inputs, ImmutableList.of());
}
+ protected FileSystem createActionFileSystem() throws IOException {
+ return createActionFileSystem(new ActionInputMap(0));
+ }
+
protected abstract FileSystem getLocalFileSystem(FileSystem actionFs);
protected abstract FileSystem getRemoteFileSystem(FileSystem actionFs);
@@ -51,8 +61,9 @@
protected abstract void writeLocalFile(FileSystem actionFs, PathFragment path, String content)
throws IOException;
- protected abstract void injectRemoteFile(FileSystem actionFs, PathFragment path, String content)
- throws IOException;
+ @CanIgnoreReturnValue
+ protected abstract FileArtifactValue injectRemoteFile(
+ FileSystem actionFs, PathFragment path, String content) throws IOException;
@Test
public void exists_fileDoesNotExist_returnsFalse() throws Exception {
