remote: implement a naive action-scoped file system
This change accompanies d480c5f [1] in that it registers
an action-scoped filesystem that supports lazily fetching
action inputs that are remotely stored outputs of an upstream
action. While in theory any native java action in Bazel could
do arbitrary file system operations in practice this is mostly
useful for
* AbstractFileWriteAction (calls Path.getInputStream())
* TemplateExpansionAction (calls Path.getInputStream())
* SymlinkAction (calls Path.(isFile|isExecutable|createSymbolicLink)())
* SolibSymlinkAction (calls Path.createSymbolicLink())
For Starlark actions we found that only ctx.actions.expand_template(...)
directly interacts with the file system (via TemplateExpansionAction).
All other methods on ctx.actions create spawns internally which are
covered by d480c5f.
Progress towards #6862.
[1] https://github.com/bazelbuild/bazel/commit/d480c5f5a4f38a4053ed3e3bcc4eaef343923d2d
Closes #7926.
PiperOrigin-RevId: 241717157
diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD
index 2a84ed2..020c2cb 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD
@@ -34,6 +34,7 @@
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/remote/util",
"//src/main/java/com/google/devtools/build/lib/vfs",
+ "//src/main/java/com/google/devtools/build/lib/vfs:output_service",
"//src/main/java/com/google/devtools/common/options",
"//third_party:auth",
"//third_party:guava",
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionFileSystem.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionFileSystem.java
new file mode 100644
index 0000000..0491c2d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionFileSystem.java
@@ -0,0 +1,373 @@
+// Copyright 2019 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.remote;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.ActionInputMap;
+import com.google.devtools.build.lib.actions.FileArtifactValue;
+import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
+import com.google.devtools.build.lib.vfs.DelegateFileSystem;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+import javax.annotation.Nullable;
+
+/**
+ * This is a basic implementation and incomplete implementation of an action file system that's been
+ * tuned to what native (non-spawn) actions in Bazel currently use.
+ *
+ * <p>The implementation mostly delegates to the local file system except for the case where an
+ * action input is a remotely stored action output. Most notably {@link #getInputStream(Path)} and
+ * {@link #createSymbolicLink(Path, PathFragment)}.
+ *
+ * <p>This implementation only supports creating local action outputs.
+ */
+public class RemoteActionFileSystem extends DelegateFileSystem {
+
+ private final Path execRoot;
+ private final Path outputBase;
+ private final ActionInputMap inputArtifactData;
+ private final RemoteActionInputFetcher inputFetcher;
+
+ public RemoteActionFileSystem(
+ FileSystem localDelegate,
+ PathFragment execRootFragment,
+ String relativeOutputPath,
+ ActionInputMap inputArtifactData,
+ RemoteActionInputFetcher inputFetcher) {
+ super(localDelegate);
+ this.execRoot = getPath(Preconditions.checkNotNull(execRootFragment, "execRootFragment"));
+ this.outputBase =
+ execRoot.getRelative(Preconditions.checkNotNull(relativeOutputPath, "relativeOutputPath"));
+ this.inputArtifactData = Preconditions.checkNotNull(inputArtifactData, "inputArtifactData");
+ this.inputFetcher = Preconditions.checkNotNull(inputFetcher, "inputFetcher");
+ }
+
+ @Override
+ public String getFileSystemType(Path path) {
+ return "remoteActionFS";
+ }
+
+ @Override
+ public boolean delete(Path path) throws IOException {
+ RemoteFileArtifactValue m = getRemoteInputMetadata(path);
+ if (m != null) {
+ return super.delete(path);
+ }
+ return true;
+ }
+
+ @Override
+ protected InputStream getInputStream(Path path) throws IOException {
+ downloadFileIfRemote(path);
+ return super.getInputStream(path);
+ }
+
+ @Override
+ public void setLastModifiedTime(Path path, long newTime) throws IOException {
+ RemoteFileArtifactValue m = getRemoteInputMetadata(path);
+ if (m == null) {
+ super.setLastModifiedTime(path, newTime);
+ }
+ }
+
+ @Override
+ protected byte[] getFastDigest(Path path) throws IOException {
+ RemoteFileArtifactValue m = getRemoteInputMetadata(path);
+ if (m != null) {
+ return m.getDigest();
+ }
+ return super.getFastDigest(path);
+ }
+
+ @Override
+ protected byte[] getDigest(Path path) throws IOException {
+ RemoteFileArtifactValue m = getRemoteInputMetadata(path);
+ if (m != null) {
+ return m.getDigest();
+ }
+ return super.getDigest(path);
+ }
+
+ // -------------------- File Permissions --------------------
+
+ @Override
+ protected boolean isReadable(Path path) throws IOException {
+ FileArtifactValue m = getRemoteInputMetadata(path);
+ return m != null || super.isReadable(path);
+ }
+
+ @Override
+ protected boolean isWritable(Path path) throws IOException {
+ FileArtifactValue m = getRemoteInputMetadata(path);
+ return m != null || super.isWritable(path);
+ }
+
+ @Override
+ protected boolean isExecutable(Path path) throws IOException {
+ FileArtifactValue m = getRemoteInputMetadata(path);
+ return m != null || super.isExecutable(path);
+ }
+
+ @Override
+ protected void setReadable(Path path, boolean readable) throws IOException {
+ RemoteFileArtifactValue m = getRemoteInputMetadata(path);
+ if (m == null) {
+ super.setReadable(path, readable);
+ }
+ }
+
+ @Override
+ public void setWritable(Path path, boolean writable) throws IOException {
+ RemoteFileArtifactValue m = getRemoteInputMetadata(path);
+ if (m == null) {
+ super.setWritable(path, writable);
+ }
+ }
+
+ @Override
+ protected void setExecutable(Path path, boolean executable) throws IOException {
+ RemoteFileArtifactValue m = getRemoteInputMetadata(path);
+ if (m == null) {
+ super.setExecutable(path, executable);
+ }
+ }
+
+ @Override
+ protected void chmod(Path path, int mode) throws IOException {
+ RemoteFileArtifactValue m = getRemoteInputMetadata(path);
+ if (m == null) {
+ super.chmod(path, mode);
+ }
+ }
+
+ // -------------------- Symlinks --------------------
+
+ @Override
+ protected PathFragment readSymbolicLink(Path path) throws IOException {
+ FileArtifactValue m = getRemoteInputMetadata(path);
+ if (m != null) {
+ // We don't support symlinks as remote action outputs.
+ throw new IOException(path + " is not a symbolic link");
+ }
+ return super.readSymbolicLink(path);
+ }
+
+ @Override
+ protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) throws IOException {
+ /*
+ * TODO(buchgr): Optimize the case where we are creating a symlink to a remote output. This does
+ * add a non-trivial amount of complications though (as symlinks tend to do).
+ */
+ downloadFileIfRemote(execRoot.getRelative(targetFragment));
+ super.createSymbolicLink(linkPath, targetFragment);
+ }
+
+ // -------------------- Implementations based on stat() --------------------
+
+ @Override
+ protected long getLastModifiedTime(Path path, boolean followSymlinks) throws IOException {
+ FileStatus stat = stat(path, followSymlinks);
+ return stat.getLastModifiedTime();
+ }
+
+ @Override
+ protected long getFileSize(Path path, boolean followSymlinks) throws IOException {
+ FileStatus stat = stat(path, followSymlinks);
+ return stat.getSize();
+ }
+
+ @Override
+ protected boolean isFile(Path path, boolean followSymlinks) {
+ FileStatus stat = statNullable(path, followSymlinks);
+ return stat != null && stat.isFile();
+ }
+
+ @Override
+ protected boolean isSymbolicLink(Path path) {
+ FileStatus stat = statNullable(path, /* followSymlinks= */ false);
+ return stat != null && stat.isSymbolicLink();
+ }
+
+ @Override
+ protected boolean isDirectory(Path path, boolean followSymlinks) {
+ FileStatus stat = statNullable(path, followSymlinks);
+ return stat != null && stat.isDirectory();
+ }
+
+ @Override
+ protected boolean isSpecialFile(Path path, boolean followSymlinks) {
+ FileStatus stat = statNullable(path, followSymlinks);
+ return stat != null && stat.isDirectory();
+ }
+
+ @Override
+ protected boolean exists(Path path, boolean followSymlinks) {
+ try {
+ return statIfFound(path, followSymlinks) != null;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public boolean exists(Path path) {
+ return exists(path, /* followSymlinks= */ true);
+ }
+
+ @Nullable
+ @Override
+ protected FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException {
+ try {
+ return stat(path, followSymlinks);
+ } catch (FileNotFoundException e) {
+ return null;
+ }
+ }
+
+ @Nullable
+ @Override
+ protected FileStatus statNullable(Path path, boolean followSymlinks) {
+ try {
+ return stat(path, followSymlinks);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ @Override
+ protected FileStatus stat(Path path, boolean followSymlinks) throws IOException {
+ RemoteFileArtifactValue m = getRemoteInputMetadata(path);
+ if (m != null) {
+ return statFromRemoteMetadata(m);
+ }
+ return super.stat(path, followSymlinks);
+ }
+
+ private FileStatus statFromRemoteMetadata(RemoteFileArtifactValue m) {
+ return new FileStatus() {
+ @Override
+ public boolean isFile() {
+ return m.getType().isFile();
+ }
+
+ @Override
+ public boolean isDirectory() {
+ return m.getType().isDirectory();
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ return m.getType().isSymlink();
+ }
+
+ @Override
+ public boolean isSpecialFile() {
+ return m.getType().isSpecialFile();
+ }
+
+ @Override
+ public long getSize() {
+ return m.getSize();
+ }
+
+ @Override
+ public long getLastModifiedTime() {
+ return m.getModifiedTime();
+ }
+
+ @Override
+ public long getLastChangeTime() {
+ return m.getModifiedTime();
+ }
+
+ @Override
+ public long getNodeId() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ // -------------------- Implementation Helpers --------------------
+
+ private String execPathString(Path path) {
+ return path.relativeTo(execRoot).getPathString();
+ }
+
+ @Nullable
+ private RemoteFileArtifactValue getRemoteInputMetadata(Path path) {
+ if (!path.startsWith(outputBase)) {
+ return null;
+ }
+ return getRemoteInputMetadata(execPathString(path));
+ }
+
+ @Nullable
+ private RemoteFileArtifactValue getRemoteInputMetadata(String execPathString) {
+ FileArtifactValue m = inputArtifactData.getMetadata(execPathString);
+ if (m != null && m.isRemote()) {
+ return (RemoteFileArtifactValue) m;
+ }
+ return null;
+ }
+
+ private void downloadFileIfRemote(Path path) throws IOException {
+ FileArtifactValue m = getRemoteInputMetadata(path);
+ if (m != null) {
+ try {
+ inputFetcher.downloadFile(toDelegatePath(path), m);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException(
+ String.format("Received interrupt while fetching file '%s'", path), e);
+ }
+ }
+ }
+
+ /*
+ * -------------------- TODO(buchgr): Not yet implemented --------------------
+ *
+ * The below methods have not (yet) been properly implemented due to time constraints mostly and
+ * with little risk as they currently don't seem to be used by native actions in Bazel. However,
+ * before making the --experimental_remote_download_outputs flag non-experimental we should make
+ * sure to fully implement this file system.
+ */
+
+ @Override
+ protected Collection<String> getDirectoryEntries(Path path) throws IOException {
+ return super.getDirectoryEntries(path);
+ }
+
+ @Override
+ protected void createFSDependentHardLink(Path linkPath, Path originalPath) throws IOException {
+ super.createFSDependentHardLink(linkPath, originalPath);
+ }
+
+ @Override
+ protected Collection<Dirent> readdir(Path path, boolean followSymlinks) throws IOException {
+ return super.readdir(path, followSymlinks);
+ }
+
+ @Override
+ protected void createHardLink(Path linkPath, Path originalPath) throws IOException {
+ super.createHardLink(linkPath, originalPath);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java
index 4eb4e2a..7f3960a 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java
@@ -13,9 +13,14 @@
// limitations under the License.
package com.google.devtools.build.lib.remote;
+import build.bazel.remote.execution.v2.Digest;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputPrefetcher;
import com.google.devtools.build.lib.actions.FileArtifactValue;
@@ -33,6 +38,9 @@
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import javax.annotation.concurrent.GuardedBy;
/**
@@ -43,13 +51,17 @@
*/
class RemoteActionInputFetcher implements ActionInputPrefetcher {
+ private static final Logger logger = Logger.getLogger(RemoteActionInputFetcher.class.getName());
+
private final Object lock = new Object();
+ /** Set of successfully downloaded output files. */
@GuardedBy("lock")
private final Set<Path> downloadedPaths = new HashSet<>();
+ @VisibleForTesting
@GuardedBy("lock")
- private final Map<Path, ListenableFuture<Void>> downloadsInProgress = new HashMap<>();
+ final Map<Path, ListenableFuture<Void>> downloadsInProgress = new HashMap<>();
private final AbstractRemoteActionCache remoteCache;
private final Path execRoot;
@@ -95,19 +107,7 @@
if (downloadedPaths.contains(path)) {
continue;
}
-
- ListenableFuture<Void> download = downloadsInProgress.get(path);
- if (download == null) {
- Context prevCtx = ctx.attach();
- try {
- download =
- remoteCache.downloadFile(
- path, DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize()));
- downloadsInProgress.put(path, download);
- } finally {
- ctx.detach(prevCtx);
- }
- }
+ ListenableFuture<Void> download = downloadFileAsync(path, metadata);
downloadsToWaitFor.putIfAbsent(path, download);
}
}
@@ -115,32 +115,22 @@
IOException ioException = null;
InterruptedException interruptedException = null;
- try {
- for (Map.Entry<Path, ListenableFuture<Void>> entry : downloadsToWaitFor.entrySet()) {
- try {
- Utils.getFromFuture(entry.getValue());
- entry.getKey().setExecutable(true);
- } catch (IOException e) {
- if (e instanceof CacheNotFoundException) {
- e =
- new IOException(
- String.format(
- "Failed to fetch file with hash '%s' because it does not exist remotely."
- + " --experimental_remote_download_outputs=minimal does not work if"
- + " your remote cache evicts files during builds.",
- ((CacheNotFoundException) e).getMissingDigest().getHash()));
- }
- ioException = ioException == null ? e : ioException;
- } catch (InterruptedException e) {
- interruptedException = interruptedException == null ? e : interruptedException;
+ for (Map.Entry<Path, ListenableFuture<Void>> entry : downloadsToWaitFor.entrySet()) {
+ try {
+ Utils.getFromFuture(entry.getValue());
+ } catch (IOException e) {
+ if (e instanceof CacheNotFoundException) {
+ e =
+ new IOException(
+ String.format(
+ "Failed to fetch file with hash '%s' because it does not exist remotely."
+ + " --experimental_remote_outputs=minimal does not work if"
+ + " your remote cache evicts files during builds.",
+ ((CacheNotFoundException) e).getMissingDigest().getHash()));
}
- }
- } finally {
- synchronized (lock) {
- for (Path path : downloadsToWaitFor.keySet()) {
- downloadsInProgress.remove(path);
- downloadedPaths.add(path);
- }
+ ioException = ioException == null ? e : ioException;
+ } catch (InterruptedException e) {
+ interruptedException = interruptedException == null ? e : interruptedException;
}
}
@@ -158,4 +148,72 @@
return ImmutableSet.copyOf(downloadedPaths);
}
}
+
+ void downloadFile(Path path, FileArtifactValue metadata)
+ throws IOException, InterruptedException {
+ try {
+ downloadFileAsync(path, metadata).get();
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw new IOException(e.getCause());
+ }
+ }
+
+ private ListenableFuture<Void> downloadFileAsync(Path path, FileArtifactValue metadata)
+ throws IOException {
+ synchronized (lock) {
+ if (downloadedPaths.contains(path)) {
+ return Futures.immediateFuture(null);
+ }
+
+ ListenableFuture<Void> download = downloadsInProgress.get(path);
+ if (download == null) {
+ Context prevCtx = ctx.attach();
+ try {
+ Digest digest = DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize());
+ download = remoteCache.downloadFile(path, digest);
+ downloadsInProgress.put(path, download);
+ Futures.addCallback(
+ download,
+ new FutureCallback<Void>() {
+ @Override
+ public void onSuccess(Void v) {
+ synchronized (lock) {
+ downloadsInProgress.remove(path);
+ downloadedPaths.add(path);
+ }
+
+ try {
+ path.setReadable(true);
+ path.setExecutable(true);
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed to chmod +xr on " + path, e);
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ synchronized (lock) {
+ downloadsInProgress.remove(path);
+ }
+ try {
+ path.delete();
+ } catch (IOException e) {
+ logger.log(
+ Level.WARNING,
+ "Failed to delete output file after incomplete download: " + path,
+ e);
+ }
+ }
+ },
+ MoreExecutors.directExecutor());
+ } finally {
+ ctx.detach(prevCtx);
+ }
+ }
+ return download;
+ }
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
index 4f3732c..35d05d6 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -43,6 +43,7 @@
import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.util.io.AsynchronousFileOutputStream;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
+import com.google.devtools.build.lib.vfs.OutputService;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.common.options.OptionsBase;
import com.google.devtools.common.options.OptionsParsingResult;
@@ -75,6 +76,8 @@
private RemoteActionContextProvider actionContextProvider;
private RemoteActionInputFetcher actionInputFetcher;
+ private RemoteOutputsMode remoteOutputsMode;
+ private RemoteOutputService remoteOutputService;
private final BuildEventArtifactUploaderFactoryDelegate
buildEventArtifactUploaderFactoryDelegate = new BuildEventArtifactUploaderFactoryDelegate();
@@ -126,6 +129,7 @@
public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
Preconditions.checkState(actionContextProvider == null, "actionContextProvider must be null");
Preconditions.checkState(actionInputFetcher == null, "actionInputFetcher must be null");
+ Preconditions.checkState(remoteOutputsMode == null, "remoteOutputsMode must be null");
RemoteOptions remoteOptions = env.getOptions().getOptions(RemoteOptions.class);
if (remoteOptions == null) {
@@ -133,6 +137,8 @@
return;
}
+ remoteOutputsMode = remoteOptions.remoteOutputsMode;
+
AuthAndTLSOptions authAndTlsOptions = env.getOptions().getOptions(AuthAndTLSOptions.class);
DigestHashFunction hashFn = env.getRuntime().getFileSystem().getDigestFunction();
DigestUtil digestUtil = new DigestUtil(hashFn);
@@ -361,6 +367,8 @@
buildEventArtifactUploaderFactoryDelegate.reset();
actionContextProvider = null;
actionInputFetcher = null;
+ remoteOutputsMode = null;
+ remoteOutputService = null;
if (failure != null) {
throw new AbruptExitException(ExitCode.LOCAL_ENVIRONMENTAL_ERROR, failure);
@@ -404,6 +412,8 @@
@Override
public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorBuilder builder) {
Preconditions.checkState(actionInputFetcher == null, "actionInputFetcher must be null");
+ Preconditions.checkNotNull(remoteOutputsMode, "remoteOutputsMode must not be null");
+
if (actionContextProvider == null) {
return;
}
@@ -420,10 +430,20 @@
new RemoteActionInputFetcher(
actionContextProvider.getRemoteCache(), env.getExecRoot(), ctx);
builder.setActionInputPrefetcher(actionInputFetcher);
+ remoteOutputService.setActionInputFetcher(actionInputFetcher);
}
}
@Override
+ public OutputService getOutputService() {
+ Preconditions.checkState(remoteOutputService == null, "remoteOutputService must be null");
+ if (remoteOutputsMode != null && !remoteOutputsMode.downloadAllOutputs()) {
+ remoteOutputService = new RemoteOutputService();
+ }
+ return remoteOutputService;
+ }
+
+ @Override
public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
return "build".equals(command.name())
? ImmutableList.of(RemoteOptions.class, AuthAndTLSOptions.class)
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java
new file mode 100644
index 0000000..29a3ef1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteOutputService.java
@@ -0,0 +1,119 @@
+// Copyright 2019 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.remote;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionInputMap;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Artifact.SourceArtifact;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.vfs.BatchStat;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.OutputService;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Root;
+import java.util.UUID;
+import java.util.function.Function;
+import javax.annotation.Nullable;
+
+/** Output service implementation for the remote module */
+public class RemoteOutputService implements OutputService {
+
+ @Nullable private RemoteActionInputFetcher actionInputFetcher;
+
+ void setActionInputFetcher(RemoteActionInputFetcher actionInputFetcher) {
+ this.actionInputFetcher = Preconditions.checkNotNull(actionInputFetcher, "actionInputFetcher");
+ }
+
+ @Override
+ public ActionFileSystemType actionFileSystemType() {
+ return actionInputFetcher != null
+ ? ActionFileSystemType.STAGE_REMOTE_FILES
+ : ActionFileSystemType.DISABLED;
+ }
+
+ @Nullable
+ @Override
+ public FileSystem createActionFileSystem(
+ FileSystem sourceDelegate,
+ PathFragment execRootFragment,
+ String relativeOutputPath,
+ ImmutableList<Root> sourceRoots,
+ ActionInputMap inputArtifactData,
+ Iterable<Artifact> outputArtifacts,
+ Function<PathFragment, SourceArtifact> sourceArtifactFactory) {
+ Preconditions.checkNotNull(actionInputFetcher, "actionInputFetcher");
+ return new RemoteActionFileSystem(
+ sourceDelegate,
+ execRootFragment,
+ relativeOutputPath,
+ inputArtifactData,
+ actionInputFetcher);
+ }
+
+ @Override
+ public String getFilesSystemName() {
+ return "remoteActionFS";
+ }
+
+ @Override
+ public ModifiedFileSet startBuild(
+ EventHandler eventHandler, UUID buildId, boolean finalizeActions) {
+ return ModifiedFileSet.EVERYTHING_MODIFIED;
+ }
+
+ @Override
+ public void finalizeBuild(boolean buildSuccessful) {
+ // Intentionally left empty.
+ }
+
+ @Override
+ public void finalizeAction(Action action, MetadataHandler metadataHandler) {
+ // Intentionally left empty.
+ }
+
+ @Nullable
+ @Override
+ public BatchStat getBatchStatter() {
+ return null;
+ }
+
+ @Override
+ public boolean canCreateSymlinkTree() {
+ /* TODO(buchgr): Optimize symlink creation for remote execution */
+ return false;
+ }
+
+ @Override
+ public void createSymlinkTree(
+ Path inputManifest, Path outputManifest, boolean filesetTree, PathFragment symlinkTreeRoot) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void clean() {
+ // Intentionally left empty.
+ }
+
+ @Override
+ public boolean isRemoteFile(Artifact file) {
+ return false;
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTest.java
new file mode 100644
index 0000000..e3c5a3b
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionFileSystemTest.java
@@ -0,0 +1,158 @@
+// Copyright 2019 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.remote;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+import com.google.common.hash.HashCode;
+import com.google.common.util.concurrent.Futures;
+import com.google.devtools.build.lib.actions.ActionInputMap;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactRoot;
+import com.google.devtools.build.lib.actions.FileArtifactValue;
+import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
+import com.google.devtools.build.lib.clock.JavaClock;
+import com.google.devtools.build.lib.vfs.DigestHashFunction;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/** Tests for {@link RemoteActionFileSystem} */
+@RunWith(JUnit4.class)
+public class RemoteActionFileSystemTest {
+
+ private static final DigestHashFunction HASH_FUNCTION = DigestHashFunction.SHA256;
+
+ @Mock private RemoteActionInputFetcher inputFetcher;
+ private FileSystem fs;
+ private Path execRoot;
+ private ArtifactRoot outputRoot;
+
+ @Before
+ public void setUp() throws IOException {
+ MockitoAnnotations.initMocks(this);
+ fs = new InMemoryFileSystem(new JavaClock(), HASH_FUNCTION);
+ execRoot = fs.getPath("/exec");
+ outputRoot = ArtifactRoot.asDerivedRoot(execRoot, execRoot.getRelative("out"));
+ outputRoot.getRoot().asPath().createDirectoryAndParents();
+ }
+
+ @Test
+ public void testGetInputStream() throws Exception {
+ // arrange
+ ActionInputMap inputs = new ActionInputMap(2);
+ Artifact remoteArtifact = createRemoteArtifact("remote-file", "remote contents", inputs);
+ Artifact localArtifact = createLocalArtifact("local-file", "local contents", inputs);
+ FileSystem actionFs = newRemoteActionFileSystem(inputs);
+ doAnswer(
+ invocationOnMock -> {
+ FileSystemUtils.writeContent(
+ remoteArtifact.getPath(), StandardCharsets.UTF_8, "remote contents");
+ return Futures.immediateFuture(null);
+ })
+ .when(inputFetcher)
+ .downloadFile(eq(remoteArtifact.getPath()), eq(inputs.getMetadata(remoteArtifact)));
+
+ // act
+ Path remoteActionFsPath = actionFs.getPath(remoteArtifact.getPath().asFragment());
+ String actualRemoteContents =
+ FileSystemUtils.readContent(remoteActionFsPath, StandardCharsets.UTF_8);
+
+ // assert
+ Path localActionFsPath = actionFs.getPath(localArtifact.getPath().asFragment());
+ String actualLocalContents =
+ FileSystemUtils.readContent(localActionFsPath, StandardCharsets.UTF_8);
+ assertThat(remoteActionFsPath.getFileSystem()).isSameAs(actionFs);
+ assertThat(actualRemoteContents).isEqualTo("remote contents");
+ assertThat(actualLocalContents).isEqualTo("local contents");
+ verify(inputFetcher)
+ .downloadFile(eq(remoteArtifact.getPath()), eq(inputs.getMetadata(remoteArtifact)));
+ verifyNoMoreInteractions(inputFetcher);
+ }
+
+ @Test
+ public void testCreateSymbolicLink() throws InterruptedException, IOException {
+ // arrange
+ ActionInputMap inputs = new ActionInputMap(1);
+ Artifact remoteArtifact = createRemoteArtifact("remote-file", "remote contents", inputs);
+ Path symlink = outputRoot.getRoot().getRelative("symlink");
+ FileSystem actionFs = newRemoteActionFileSystem(inputs);
+ doAnswer(
+ invocationOnMock -> {
+ FileSystemUtils.writeContent(
+ remoteArtifact.getPath(), StandardCharsets.UTF_8, "remote contents");
+ return Futures.immediateFuture(null);
+ })
+ .when(inputFetcher)
+ .downloadFile(eq(remoteArtifact.getPath()), eq(inputs.getMetadata(remoteArtifact)));
+
+ // act
+ Path symlinkActionFs = actionFs.getPath(symlink.getPathString());
+ symlinkActionFs.createSymbolicLink(actionFs.getPath(remoteArtifact.getPath().asFragment()));
+
+ // assert
+ assertThat(symlinkActionFs.getFileSystem()).isSameAs(actionFs);
+ verify(inputFetcher)
+ .downloadFile(eq(remoteArtifact.getPath()), eq(inputs.getMetadata(remoteArtifact)));
+ String symlinkTargetContents =
+ FileSystemUtils.readContent(symlinkActionFs, StandardCharsets.UTF_8);
+ assertThat(symlinkTargetContents).isEqualTo("remote contents");
+ verifyNoMoreInteractions(inputFetcher);
+ }
+
+ private FileSystem newRemoteActionFileSystem(ActionInputMap inputs) {
+ return new RemoteActionFileSystem(
+ fs,
+ execRoot.asFragment(),
+ outputRoot.getRoot().asPath().relativeTo(execRoot).getPathString(),
+ inputs,
+ inputFetcher);
+ }
+
+ /** Returns a remote artifact and puts its metadata into the action input map. */
+ private Artifact createRemoteArtifact(
+ String pathFragment, String contents, ActionInputMap inputs) {
+ Path p = outputRoot.getRoot().asPath().getRelative(pathFragment);
+ Artifact a = new Artifact(p, outputRoot);
+ byte[] b = contents.getBytes(StandardCharsets.UTF_8);
+ HashCode h = HASH_FUNCTION.getHashFunction().hashBytes(b);
+ FileArtifactValue f =
+ new RemoteFileArtifactValue(h.asBytes(), b.length, /* locationIndex= */ 1);
+ inputs.putWithNoDepOwner(a, f);
+ return a;
+ }
+
+ /** Returns a local artifact and puts its metadata into the action input map. */
+ private Artifact createLocalArtifact(String pathFragment, String contents, ActionInputMap inputs)
+ throws IOException {
+ Path p = outputRoot.getRoot().asPath().getRelative(pathFragment);
+ FileSystemUtils.writeContent(p, StandardCharsets.UTF_8, contents);
+ Artifact a = new Artifact(p, outputRoot);
+ inputs.putWithNoDepOwner(a, FileArtifactValue.create(a.getPath(), /* isShareable= */ true));
+ return a;
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcherTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcherTest.java
index 8a10229..82a2a62 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcherTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcherTest.java
@@ -106,6 +106,7 @@
assertThat(a2.getPath().isExecutable()).isTrue();
assertThat(actionInputFetcher.downloadedFiles()).hasSize(2);
assertThat(actionInputFetcher.downloadedFiles()).containsAllOf(a1.getPath(), a2.getPath());
+ assertThat(actionInputFetcher.downloadsInProgress).isEmpty();
}
@Test
@@ -126,6 +127,7 @@
assertThat(FileSystemUtils.readContent(p, StandardCharsets.UTF_8)).isEqualTo("hello world");
assertThat(p.isExecutable()).isFalse();
assertThat(actionInputFetcher.downloadedFiles()).isEmpty();
+ assertThat(actionInputFetcher.downloadsInProgress).isEmpty();
}
@Test
@@ -151,7 +153,8 @@
}
// assert
- assertThat(actionInputFetcher.downloadedFiles()).containsExactly(a.getPath());
+ assertThat(actionInputFetcher.downloadedFiles()).isEmpty();
+ assertThat(actionInputFetcher.downloadsInProgress).isEmpty();
}
@Test
@@ -174,6 +177,28 @@
// assert
assertThat(actionInputFetcher.downloadedFiles()).isEmpty();
+ assertThat(actionInputFetcher.downloadsInProgress).isEmpty();
+ }
+
+ @Test
+ public void testDownloadFile() throws Exception {
+ // arrange
+ Map<ActionInput, FileArtifactValue> metadata = new HashMap<>();
+ Map<Digest, ByteString> cacheEntries = new HashMap<>();
+ Artifact a1 = createRemoteArtifact("file1", "hello world", metadata, cacheEntries);
+ AbstractRemoteActionCache remoteCache =
+ new StaticRemoteActionCache(options, digestUtil, cacheEntries);
+ RemoteActionInputFetcher actionInputFetcher =
+ new RemoteActionInputFetcher(remoteCache, execRoot, Context.current());
+
+ // act
+ actionInputFetcher.downloadFile(a1.getPath(), metadata.get(a1));
+
+ // assert
+ assertThat(FileSystemUtils.readContent(a1.getPath(), StandardCharsets.UTF_8))
+ .isEqualTo("hello world");
+ assertThat(a1.getPath().isExecutable()).isTrue();
+ assertThat(a1.getPath().isReadable()).isTrue();
}
private Artifact createRemoteArtifact(
diff --git a/src/test/shell/bazel/remote/remote_execution_test.sh b/src/test/shell/bazel/remote/remote_execution_test.sh
index 84be7f1..9d1494d 100755
--- a/src/test/shell/bazel/remote/remote_execution_test.sh
+++ b/src/test/shell/bazel/remote/remote_execution_test.sh
@@ -937,6 +937,69 @@
expect_log "1 process: 1 remote"
}
+function test_downloads_minimal_native_prefetch() {
+ # Test that when using --experimental_remote_outputs=minimal a remotely stored output that's an
+ # input to a native action (ctx.actions.expand_template) is staged lazily for action execution.
+ mkdir -p a
+ cat > a/substitute_username.bzl <<'EOF'
+def _substitute_username_impl(ctx):
+ ctx.actions.expand_template(
+ template = ctx.file.template,
+ output = ctx.outputs.out,
+ substitutions = {
+ "{USERNAME}": ctx.attr.username,
+ },
+ )
+
+substitute_username = rule(
+ implementation = _substitute_username_impl,
+ attrs = {
+ "username": attr.string(mandatory = True),
+ "template": attr.label(
+ allow_single_file = True,
+ mandatory = True,
+ ),
+ },
+ outputs = {"out": "%{name}.txt"},
+)
+EOF
+
+ cat > a/BUILD <<'EOF'
+load(":substitute_username.bzl", "substitute_username")
+genrule(
+ name = "generate-template",
+ cmd = "echo -n \"Hello {USERNAME}!\" > $@",
+ outs = ["template.txt"],
+ srcs = [],
+)
+
+substitute_username(
+ name = "substitute-buchgr",
+ username = "buchgr",
+ template = ":generate-template",
+)
+EOF
+
+ bazel build \
+ --genrule_strategy=remote \
+ --remote_executor=localhost:${worker_port} \
+ --experimental_inmemory_jdeps_files \
+ --experimental_inmemory_dotd_files \
+ --experimental_remote_download_outputs=minimal \
+ //a:substitute-buchgr >& $TEST_log || fail "Failed to build //a:substitute-buchgr"
+
+ # The genrule //a:generate-template should run remotely and //a:substitute-buchgr
+ # should be a native action running locally.
+ expect_log "1 process: 1 remote"
+
+ outtxt="bazel-bin/a/substitute-buchgr.txt"
+ [[ $(< ${outtxt}) == "Hello buchgr!" ]] \
+ || fail "Unexpected contents in "${outtxt}":" $(< ${outtxt})
+
+ (! [[ -f bazel-bin/a/template.txt ]]) \
+ || fail "Expected bazel-bin/a/template.txt to have been deleted again"
+}
+
# TODO(alpha): Add a test that fails remote execution when remote worker
# supports sandbox.
