remote: add action input fetcher
Add a RemoteActionInputFetcher class which can stage action inputs that
are only available on a remote system. This change only introduces the
class and tests. Will enable it in a follow up CL.
Progress towards #6862
Closes #7866.
PiperOrigin-RevId: 240953164
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java
new file mode 100644
index 0000000..8e6470d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java
@@ -0,0 +1,161 @@
+// Copyright 2019 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.remote;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputPrefetcher;
+import com.google.devtools.build.lib.actions.FileArtifactValue;
+import com.google.devtools.build.lib.actions.MetadataProvider;
+import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.SilentCloseable;
+import com.google.devtools.build.lib.remote.util.DigestUtil;
+import com.google.devtools.build.lib.remote.util.Utils;
+import com.google.devtools.build.lib.vfs.Path;
+import io.grpc.Context;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import javax.annotation.concurrent.GuardedBy;
+
+/**
+ * Stages output files that are stored remotely to the local filesystem.
+ *
+ * <p>This is necessary for remote caching/execution when {@code
+ * --experimental_remote_fetch_outputs=minimal} is specified.
+ */
+class RemoteActionInputFetcher implements ActionInputPrefetcher {
+
+ private final Object lock = new Object();
+
+ @GuardedBy("lock")
+ private final Set<Path> downloadedPaths = new HashSet<>();
+
+ @GuardedBy("lock")
+ private final Map<Path, ListenableFuture<Void>> downloadsInProgress = new HashMap<>();
+
+ private final AbstractRemoteActionCache remoteCache;
+ private final Path execRoot;
+ private final Context ctx;
+
+ RemoteActionInputFetcher(AbstractRemoteActionCache remoteCache, Path execRoot, Context ctx) {
+ this.remoteCache = Preconditions.checkNotNull(remoteCache);
+ this.execRoot = Preconditions.checkNotNull(execRoot);
+ this.ctx = Preconditions.checkNotNull(ctx);
+ }
+
+ /**
+ * Fetches remotely stored action outputs, that are inputs to this spawn, and stores them under
+ * their path in the output base.
+ *
+ * <p>This method blocks until all downloads have finished.
+ *
+ * <p>This method is safe to be called concurrently from spawn runners before running any local
+ * spawn.
+ */
+ @Override
+ public void prefetchFiles(
+ Iterable<? extends ActionInput> inputs, MetadataProvider metadataProvider)
+ throws IOException, InterruptedException {
+ try (SilentCloseable c = Profiler.instance().profile("Remote.fetchInputs")) {
+ Map<Path, ListenableFuture<Void>> downloadsToWaitFor = new HashMap<>();
+ for (ActionInput input : inputs) {
+ if (input instanceof VirtualActionInput) {
+ VirtualActionInput paramFileActionInput = (VirtualActionInput) input;
+ Path outputPath = execRoot.getRelative(paramFileActionInput.getExecPath());
+ outputPath.getParentDirectory().createDirectoryAndParents();
+ try (OutputStream out = outputPath.getOutputStream()) {
+ paramFileActionInput.writeTo(out);
+ }
+ } else {
+ FileArtifactValue metadata = metadataProvider.getMetadata(input);
+ if (metadata == null || !metadata.isRemote()) {
+ continue;
+ }
+
+ Path path = execRoot.getRelative(input.getExecPath());
+ synchronized (lock) {
+ if (downloadedPaths.contains(path)) {
+ continue;
+ }
+
+ ListenableFuture<Void> download = downloadsInProgress.get(path);
+ if (download == null) {
+ Context prevCtx = ctx.attach();
+ try {
+ download =
+ remoteCache.downloadFile(
+ path, DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize()));
+ downloadsInProgress.put(path, download);
+ } finally {
+ ctx.detach(prevCtx);
+ }
+ }
+ downloadsToWaitFor.putIfAbsent(path, download);
+ }
+ }
+ }
+
+ IOException ioException = null;
+ InterruptedException interruptedException = null;
+ try {
+ for (Map.Entry<Path, ListenableFuture<Void>> entry : downloadsToWaitFor.entrySet()) {
+ try {
+ Utils.getFromFuture(entry.getValue());
+ entry.getKey().setExecutable(true);
+ } catch (IOException e) {
+ if (e instanceof CacheNotFoundException) {
+ e =
+ new IOException(
+ String.format(
+ "Failed to fetch file with hash '%s' because it does not exist remotely."
+ + " --experimental_remote_fetch_outputs=minimal does not work if"
+ + " your remote cache evicts files during builds.",
+ ((CacheNotFoundException) e).getMissingDigest().getHash()));
+ }
+ ioException = ioException == null ? e : ioException;
+ } catch (InterruptedException e) {
+ interruptedException = interruptedException == null ? e : interruptedException;
+ }
+ }
+ } finally {
+ synchronized (lock) {
+ for (Path path : downloadsToWaitFor.keySet()) {
+ downloadsInProgress.remove(path);
+ downloadedPaths.add(path);
+ }
+ }
+ }
+
+ if (interruptedException != null) {
+ throw interruptedException;
+ }
+ if (ioException != null) {
+ throw ioException;
+ }
+ }
+ }
+
+ ImmutableSet<Path> downloadedFiles() {
+ synchronized (lock) {
+ return ImmutableSet.copyOf(downloadedPaths);
+ }
+ }
+}
