Remove contains() and containsActionResult from SimpleBlobStore
contains() and containsActionResult() only make sense for the OnDiskBlobStore
because they can be implemented with a (quickly returning) stat() call. They are
not generally useful for remote caching/execution. Thus this change
moves them down to the OnDiskBlobStore.
In the same breath I removed any non-test uses of the InMemoryBlobStore
and moved the class to the test package. The InMemoryBlobStore has never
been used by Bazel itself but only by its tests and the LRE. The LRE
usage has never really made sense either and so I removed it from there
as well.
Closes #9173.
PiperOrigin-RevId: 264590465
diff --git a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java
index 8822c0d..2796f0f 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCache.java
@@ -48,8 +48,8 @@
* <p>Note that this class is used from src/tools/remote.
*/
@ThreadSafe
-public final class SimpleBlobStoreActionCache extends AbstractRemoteActionCache {
- private final SimpleBlobStore blobStore;
+public class SimpleBlobStoreActionCache extends AbstractRemoteActionCache {
+ protected final SimpleBlobStore blobStore;
public SimpleBlobStoreActionCache(
RemoteOptions options, SimpleBlobStore blobStore, DigestUtil digestUtil) {
@@ -86,10 +86,6 @@
return ImmutableSet.copyOf(digests);
}
- public boolean containsKey(Digest digest) throws IOException, InterruptedException {
- return blobStore.contains(digest.getHash());
- }
-
@Override
public ActionResult getCachedActionResult(ActionKey actionKey)
throws IOException, InterruptedException {
diff --git a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java
index f006f52..89a3bbc 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactory.java
@@ -18,7 +18,6 @@
import com.google.common.base.Ascii;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
-import com.google.devtools.build.lib.remote.blobstore.ConcurrentMapBlobStore;
import com.google.devtools.build.lib.remote.common.SimpleBlobStore;
import com.google.devtools.build.lib.remote.disk.CombinedDiskHttpBlobStore;
import com.google.devtools.build.lib.remote.disk.OnDiskBlobStore;
@@ -29,7 +28,6 @@
import io.netty.channel.unix.DomainSocketAddress;
import java.io.IOException;
import java.net.URI;
-import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.Nullable;
/**
@@ -46,7 +44,7 @@
} else if (casPath != null) {
return new OnDiskBlobStore(casPath);
} else {
- return new ConcurrentMapBlobStore(new ConcurrentHashMap<>());
+ return null;
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/blobstore/ConcurrentMapBlobStore.java b/src/main/java/com/google/devtools/build/lib/remote/blobstore/ConcurrentMapBlobStore.java
index 57074e3..2aaa588 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/blobstore/ConcurrentMapBlobStore.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/blobstore/ConcurrentMapBlobStore.java
@@ -13,93 +13,5 @@
// limitations under the License.
package com.google.devtools.build.lib.remote.blobstore;
-import build.bazel.remote.execution.v2.ActionResult;
-import build.bazel.remote.execution.v2.Digest;
-import com.google.common.base.Preconditions;
-import com.google.common.io.ByteStreams;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-import com.google.devtools.build.lib.remote.common.SimpleBlobStore;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.protobuf.ByteString;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.concurrent.ConcurrentMap;
/** A {@link SimpleBlobStore} implementation using a {@link ConcurrentMap}. */
-public final class ConcurrentMapBlobStore implements SimpleBlobStore {
- private final ConcurrentMap<String, byte[]> map;
- static final String ACTION_KEY_PREFIX = "ac_";
-
- public ConcurrentMapBlobStore(ConcurrentMap<String, byte[]> map) {
- this.map = map;
- }
-
- @Override
- public boolean contains(String key) {
- return map.containsKey(key);
- }
-
- @Override
- public boolean containsActionResult(String key) {
- return map.containsKey(ACTION_KEY_PREFIX + key);
- }
-
- @Override
- public ListenableFuture<Boolean> get(String key, OutputStream out) {
- byte[] data = map.get(key);
- SettableFuture<Boolean> f = SettableFuture.create();
- if (data == null) {
- f.set(false);
- } else {
- try {
- out.write(data);
- f.set(true);
- } catch (IOException e) {
- f.setException(e);
- }
- }
- return f;
- }
-
- @Override
- public ListenableFuture<Boolean> getActionResult(String key, OutputStream out) {
- return get(ACTION_KEY_PREFIX + key, out);
- }
-
- @Override
- public void putActionResult(ActionKey actionKey, ActionResult actionResult) {
- map.put(ACTION_KEY_PREFIX + actionKey.getDigest().getHash(), actionResult.toByteArray());
- }
-
- @Override
- public void close() {}
-
- @Override
- public ListenableFuture<Void> uploadFile(Digest digest, Path file) {
- try (InputStream in = file.getInputStream()) {
- upload(digest.getHash(), digest.getSizeBytes(), in);
- } catch (IOException e) {
- return Futures.immediateFailedFuture(e);
- }
- return Futures.immediateFuture(null);
- }
-
- @Override
- public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
- try (InputStream in = data.newInput()) {
- upload(digest.getHash(), digest.getSizeBytes(), in);
- } catch (IOException e) {
- return Futures.immediateFailedFuture(e);
- }
- return Futures.immediateFuture(null);
- }
-
- private void upload(String key, long length, InputStream in) throws IOException {
- byte[] value = ByteStreams.toByteArray(in);
- Preconditions.checkState(value.length == length);
- map.put(key, value);
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/common/SimpleBlobStore.java b/src/main/java/com/google/devtools/build/lib/remote/common/SimpleBlobStore.java
index 3bea63e..92af8c7 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/common/SimpleBlobStore.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/common/SimpleBlobStore.java
@@ -45,12 +45,6 @@
}
}
- /** Returns {@code true} if the provided {@code key} is stored in the CAS. */
- boolean contains(String key) throws IOException, InterruptedException;
-
- /** Returns {@code true} if the provided {@code key} is stored in the Action Cache. */
- boolean containsActionResult(String key) throws IOException, InterruptedException;
-
/**
* Fetches the BLOB associated with the {@code key} from the CAS and writes it to {@code out}.
*
diff --git a/src/main/java/com/google/devtools/build/lib/remote/disk/CombinedDiskHttpBlobStore.java b/src/main/java/com/google/devtools/build/lib/remote/disk/CombinedDiskHttpBlobStore.java
index 2c85775..c9377e4 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/disk/CombinedDiskHttpBlobStore.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/disk/CombinedDiskHttpBlobStore.java
@@ -46,16 +46,6 @@
}
@Override
- public boolean contains(String key) {
- return diskCache.contains(key);
- }
-
- @Override
- public boolean containsActionResult(String key) {
- return diskCache.containsActionResult(key);
- }
-
- @Override
public void putActionResult(ActionKey actionKey, ActionResult actionResult)
throws IOException, InterruptedException {
diskCache.putActionResult(actionKey, actionResult);
diff --git a/src/main/java/com/google/devtools/build/lib/remote/disk/OnDiskBlobStore.java b/src/main/java/com/google/devtools/build/lib/remote/disk/OnDiskBlobStore.java
index eaaaeac..205dd9f 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/disk/OnDiskBlobStore.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/disk/OnDiskBlobStore.java
@@ -36,12 +36,12 @@
this.root = root;
}
- @Override
+ /** Returns {@code true} if the provided {@code key} is stored in the CAS. */
public boolean contains(String key) {
return toPath(key, /* actionResult= */ false).exists();
}
- @Override
+ /** Returns {@code true} if the provided {@code key} is stored in the Action Cache. */
public boolean containsActionResult(String key) {
return toPath(key, /* actionResult= */ true).exists();
}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/HttpBlobStore.java b/src/main/java/com/google/devtools/build/lib/remote/http/HttpBlobStore.java
index 3c1b16b..8065a62 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/http/HttpBlobStore.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/http/HttpBlobStore.java
@@ -416,16 +416,6 @@
}
@Override
- public boolean contains(String key) {
- throw new UnsupportedOperationException("HTTP Caching does not use this method.");
- }
-
- @Override
- public boolean containsActionResult(String key) {
- throw new UnsupportedOperationException("HTTP Caching does not use this method.");
- }
-
- @Override
public ListenableFuture<Boolean> get(String key, OutputStream out) {
return get(key, out, true);
}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCacheTest.java
index 4071de4..1e3ead4 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCacheTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/SimpleBlobStoreActionCacheTest.java
@@ -27,12 +27,17 @@
import build.bazel.remote.execution.v2.FileNode;
import build.bazel.remote.execution.v2.Tree;
import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import com.google.common.io.ByteStreams;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.SettableFuture;
import com.google.devtools.build.lib.actions.ActionInputHelper;
import com.google.devtools.build.lib.clock.JavaClock;
-import com.google.devtools.build.lib.remote.blobstore.ConcurrentMapBlobStore;
+import com.google.devtools.build.lib.remote.common.SimpleBlobStore;
import com.google.devtools.build.lib.remote.common.SimpleBlobStore.ActionKey;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.remote.util.DigestUtil;
@@ -44,8 +49,11 @@
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import com.google.devtools.common.options.Options;
+import com.google.protobuf.ByteString;
import io.grpc.Context;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -434,4 +442,69 @@
() -> getFromFuture(client.downloadFile(fs.getPath("/exec/root/foo"), digest)));
assertThat(e).hasMessageThat().contains(digest.getHash());
}
+
+ private static class ConcurrentMapBlobStore implements SimpleBlobStore {
+ private final ConcurrentMap<String, byte[]> map;
+ private static final String ACTION_KEY_PREFIX = "ac_";
+
+ public ConcurrentMapBlobStore(ConcurrentMap<String, byte[]> map) {
+ this.map = map;
+ }
+
+ @Override
+ public ListenableFuture<Boolean> get(String key, OutputStream out) {
+ byte[] data = map.get(key);
+ SettableFuture<Boolean> f = SettableFuture.create();
+ if (data == null) {
+ f.set(false);
+ } else {
+ try {
+ out.write(data);
+ f.set(true);
+ } catch (IOException e) {
+ f.setException(e);
+ }
+ }
+ return f;
+ }
+
+ @Override
+ public ListenableFuture<Boolean> getActionResult(String key, OutputStream out) {
+ return get(ACTION_KEY_PREFIX + key, out);
+ }
+
+ @Override
+ public void putActionResult(ActionKey actionKey, ActionResult actionResult) {
+ map.put(ACTION_KEY_PREFIX + actionKey.getDigest().getHash(), actionResult.toByteArray());
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public ListenableFuture<Void> uploadFile(Digest digest, Path file) {
+ try (InputStream in = file.getInputStream()) {
+ upload(digest.getHash(), digest.getSizeBytes(), in);
+ } catch (IOException e) {
+ return Futures.immediateFailedFuture(e);
+ }
+ return Futures.immediateFuture(null);
+ }
+
+ @Override
+ public ListenableFuture<Void> uploadBlob(Digest digest, ByteString data) {
+ try (InputStream in = data.newInput()) {
+ upload(digest.getHash(), digest.getSizeBytes(), in);
+ } catch (IOException e) {
+ return Futures.immediateFailedFuture(e);
+ }
+ return Futures.immediateFuture(null);
+ }
+
+ private void upload(String key, long length, InputStream in) throws IOException {
+ byte[] value = ByteStreams.toByteArray(in);
+ Preconditions.checkState(value.length == length);
+ map.put(key, value);
+ }
+ }
}
diff --git a/src/test/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactoryTest.java b/src/test/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactoryTest.java
index 96dd3ea..3b95dda 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactoryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/SimpleBlobStoreFactoryTest.java
@@ -18,7 +18,6 @@
import static com.google.devtools.build.lib.testutil.MoreAsserts.assertThrows;
import com.google.devtools.build.lib.clock.JavaClock;
-import com.google.devtools.build.lib.remote.blobstore.ConcurrentMapBlobStore;
import com.google.devtools.build.lib.remote.common.SimpleBlobStore;
import com.google.devtools.build.lib.remote.disk.CombinedDiskHttpBlobStore;
import com.google.devtools.build.lib.remote.disk.OnDiskBlobStore;
@@ -210,70 +209,4 @@
public void isRemoteCacheOptions_defaultOptions() {
assertThat(SimpleBlobStoreFactory.isRemoteCacheOptions(remoteOptions)).isFalse();
}
-
- @Test
- public void create_httpCacheWhenHttpAndDiskCacheEnabled() {
- remoteOptions.remoteCache = "http://doesnotexist.com";
- remoteOptions.diskCache = PathFragment.create("/etc/something/cache/here");
-
- SimpleBlobStore blobStore = SimpleBlobStoreFactory.create(remoteOptions, /* casPath= */ null);
-
- assertThat(blobStore).isInstanceOf(HttpBlobStore.class);
- }
-
- @Test
- public void create_httpCacheWithProxy() {
- remoteOptions.remoteCache = "http://doesnotexist.com";
- remoteOptions.remoteProxy = "unix://some-proxy";
-
- SimpleBlobStore blobStore = SimpleBlobStoreFactory.create(remoteOptions, /* casPath= */ null);
-
- assertThat(blobStore).isInstanceOf(HttpBlobStore.class);
- }
-
- @Test
- public void create_httpCacheFailsWithUnsupportedProxyProtocol() {
- remoteOptions.remoteCache = "http://doesnotexist.com";
- remoteOptions.remoteProxy = "bad-proxy";
-
- assertThat(
- assertThrows(
- Exception.class,
- () -> SimpleBlobStoreFactory.create(remoteOptions, /* casPath= */ null)))
- .hasMessageThat()
- .contains("Remote cache proxy unsupported: bad-proxy");
- }
-
- @Test
- public void create_httpCacheWithoutProxy() {
- remoteOptions.remoteCache = "http://doesnotexist.com";
-
- SimpleBlobStore blobStore = SimpleBlobStoreFactory.create(remoteOptions, /* casPath= */ null);
-
- assertThat(blobStore).isInstanceOf(HttpBlobStore.class);
- }
-
- @Test
- public void create_diskCacheWithCasPath() {
- SimpleBlobStore blobStore =
- SimpleBlobStoreFactory.create(remoteOptions, fs.getPath("/cas/path/is/here"));
-
- assertThat(blobStore).isInstanceOf(OnDiskBlobStore.class);
- }
-
- @Test
- public void create_defaultCacheWhenDiskCacheEnabled() {
- remoteOptions.diskCache = PathFragment.create("/etc/something/cache/here");
-
- SimpleBlobStore blobStore = SimpleBlobStoreFactory.create(remoteOptions, /* casPath= */ null);
-
- assertThat(blobStore).isInstanceOf(ConcurrentMapBlobStore.class);
- }
-
- @Test
- public void create_defaultCache() {
- SimpleBlobStore blobStore = SimpleBlobStoreFactory.create(remoteOptions, null);
-
- assertThat(blobStore).isInstanceOf(ConcurrentMapBlobStore.class);
- }
}
diff --git a/src/test/py/bazel/test_base.py b/src/test/py/bazel/test_base.py
index f7d6f09..3b93157 100644
--- a/src/test/py/bazel/test_base.py
+++ b/src/test/py/bazel/test_base.py
@@ -16,6 +16,7 @@
import locale
import os
+import shutil
import socket
import stat
import subprocess
@@ -49,6 +50,7 @@
_worker_stdout = None
_worker_stderr = None
_worker_proc = None
+ _cas_path = None
def setUp(self):
unittest.TestCase.setUp(self)
@@ -318,6 +320,8 @@
# path length limits, so we run straight in TEMP. This should ideally
# be set to something like C:\temp. On CI this is set to D:\temp.
worker_path = TestBase.GetEnv('TEMP')
+ self._cas_path = worker_path + '\\cas'
+ os.mkdir(self._cas_path)
# Get an open port. Unfortunately this seems to be the best option in
# Python.
@@ -334,6 +338,7 @@
# This path has to be extremely short to avoid Windows path
# length restrictions.
'--work_path=' + worker_path,
+ '--cas_path=' + self._cas_path,
],
stdout=self._worker_stdout,
stderr=self._worker_stderr,
@@ -372,6 +377,8 @@
print('--------------------------')
print('\n'.join(stderr_lines))
+ shutil.rmtree(self._cas_path)
+
def RunProgram(self,
args,
env_remove=None,
diff --git a/src/test/shell/bazel/remote/remote_execution_http_test.sh b/src/test/shell/bazel/remote/remote_execution_http_test.sh
index 3fa2676..a85df0c 100755
--- a/src/test/shell/bazel/remote/remote_execution_http_test.sh
+++ b/src/test/shell/bazel/remote/remote_execution_http_test.sh
@@ -24,6 +24,7 @@
function set_up() {
work_path=$(mktemp -d "${TEST_TMPDIR}/remote.XXXXXXXX")
+ cas_path=$(mktemp -d "${TEST_TMPDIR}/remote.XXXXXXXX")
pid_file=$(mktemp -u "${TEST_TMPDIR}/remote.XXXXXXXX")
attempts=1
while [ $attempts -le 5 ]; do
@@ -32,6 +33,7 @@
http_port=$(pick_random_unused_tcp_port) || fail "no port found"
"${BAZEL_RUNFILES}/src/tools/remote/worker" \
--work_path="${work_path}" \
+ --cas_path="${cas_path}" \
--listen_port=${worker_port} \
--http_listen_port=${http_port} \
--pid_file="${pid_file}" &
@@ -57,6 +59,7 @@
fi
rm -rf "${pid_file}"
rm -rf "${work_path}"
+ rm -rf "${cas_path}"
}
function test_remote_http_cache_flag() {
diff --git a/src/test/shell/bazel/remote/remote_execution_sandboxing_test.sh b/src/test/shell/bazel/remote/remote_execution_sandboxing_test.sh
index ad183aa..62d25bb3 100755
--- a/src/test/shell/bazel/remote/remote_execution_sandboxing_test.sh
+++ b/src/test/shell/bazel/remote/remote_execution_sandboxing_test.sh
@@ -26,12 +26,14 @@
function set_up() {
work_path=$(mktemp -d "${TEST_TMPDIR}/remote.XXXXXXXX")
+ cas_path=$(mktemp -d "${TEST_TMPDIR}/remote.XXXXXXXX")
writable_path=$(mktemp -d "${TEST_TMPDIR}/remote.XXXXXXXX")
readonly_path=$(mktemp -d "${TEST_TMPDIR}/remote.XXXXXXXX")
pid_file=$(mktemp -u "${TEST_TMPDIR}/remote.XXXXXXXX")
worker_port=$(pick_random_unused_tcp_port) || fail "no port found"
"${BAZEL_RUNFILES}/src/tools/remote/worker" \
--work_path="${work_path}" \
+ --cas_path="${cas_path}" \
--listen_port=${worker_port} \
--sandboxing \
--sandboxing_writable_path="${writable_path}" \
@@ -80,6 +82,7 @@
fi
rm -rf "${pid_file}"
rm -rf "${work_path}"
+ rm -rf "${cas_path}"
}
function test_genrule() {
diff --git a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/BUILD b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/BUILD
index e9dc09d..2a509c4 100644
--- a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/BUILD
+++ b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/BUILD
@@ -28,6 +28,7 @@
"//src/main/java/com/google/devtools/build/lib/remote",
"//src/main/java/com/google/devtools/build/lib/remote/blobstore",
"//src/main/java/com/google/devtools/build/lib/remote/common",
+ "//src/main/java/com/google/devtools/build/lib/remote/disk",
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/remote/util",
"//src/main/java/com/google/devtools/build/lib/sandbox",
diff --git a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ByteStreamServer.java b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ByteStreamServer.java
index d9b8635..b4a4b25 100644
--- a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ByteStreamServer.java
+++ b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/ByteStreamServer.java
@@ -26,7 +26,6 @@
import com.google.bytestream.ByteStreamProto.WriteResponse;
import com.google.devtools.build.lib.remote.CacheNotFoundException;
import com.google.devtools.build.lib.remote.Chunker;
-import com.google.devtools.build.lib.remote.SimpleBlobStoreActionCache;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
@@ -42,7 +41,7 @@
/** A basic implementation of a {@link ByteStreamImplBase} service. */
final class ByteStreamServer extends ByteStreamImplBase {
private static final Logger logger = Logger.getLogger(ByteStreamServer.class.getName());
- private final SimpleBlobStoreActionCache cache;
+ private final OnDiskBlobStoreActionCache cache;
private final Path workPath;
private final DigestUtil digestUtil;
@@ -61,7 +60,7 @@
}
}
- public ByteStreamServer(SimpleBlobStoreActionCache cache, Path workPath, DigestUtil digestUtil) {
+ public ByteStreamServer(OnDiskBlobStoreActionCache cache, Path workPath, DigestUtil digestUtil) {
this.cache = cache;
this.workPath = workPath;
this.digestUtil = digestUtil;
@@ -135,21 +134,10 @@
}
if (offset == 0) {
- try {
- if (cache.containsKey(digest)) {
- responseObserver.onNext(
- WriteResponse.newBuilder().setCommittedSize(digest.getSizeBytes()).build());
- responseObserver.onCompleted();
- closed = true;
- return;
- }
- } catch (InterruptedException e) {
- responseObserver.onError(StatusUtils.interruptedError(digest));
- Thread.currentThread().interrupt();
- closed = true;
- return;
- } catch (IOException e) {
- responseObserver.onError(StatusUtils.internalError(e));
+ if (cache.containsKey(digest)) {
+ responseObserver.onNext(
+ WriteResponse.newBuilder().setCommittedSize(digest.getSizeBytes()).build());
+ responseObserver.onCompleted();
closed = true;
return;
}
diff --git a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/CasServer.java b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/CasServer.java
index 45fa3c2..6cd6949 100644
--- a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/CasServer.java
+++ b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/CasServer.java
@@ -22,17 +22,15 @@
import build.bazel.remote.execution.v2.Digest;
import build.bazel.remote.execution.v2.FindMissingBlobsRequest;
import build.bazel.remote.execution.v2.FindMissingBlobsResponse;
-import com.google.devtools.build.lib.remote.SimpleBlobStoreActionCache;
import com.google.rpc.Code;
import io.grpc.stub.StreamObserver;
-import java.io.IOException;
/** A basic implementation of a {@link ContentAddressableStorageImplBase} service. */
final class CasServer extends ContentAddressableStorageImplBase {
static final long MAX_BATCH_SIZE_BYTES = 1024 * 1024 * 4;
- private final SimpleBlobStoreActionCache cache;
+ private final OnDiskBlobStoreActionCache cache;
- public CasServer(SimpleBlobStoreActionCache cache) {
+ public CasServer(OnDiskBlobStoreActionCache cache) {
this.cache = cache;
}
@@ -41,23 +39,13 @@
FindMissingBlobsRequest request, StreamObserver<FindMissingBlobsResponse> responseObserver) {
FindMissingBlobsResponse.Builder response = FindMissingBlobsResponse.newBuilder();
- try {
- for (Digest digest : request.getBlobDigestsList()) {
- try {
- if (!cache.containsKey(digest)) {
- response.addMissingBlobDigests(digest);
- }
- } catch (InterruptedException e) {
- responseObserver.onError(StatusUtils.interruptedError(digest));
- Thread.currentThread().interrupt();
- return;
- }
+ for (Digest digest : request.getBlobDigestsList()) {
+ if (!cache.containsKey(digest)) {
+ response.addMissingBlobDigests(digest);
}
- responseObserver.onNext(response.build());
- responseObserver.onCompleted();
- } catch (IOException e) {
- responseObserver.onError(StatusUtils.internalError(e));
}
+ responseObserver.onNext(response.build());
+ responseObserver.onCompleted();
}
@Override
diff --git a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/OnDiskBlobStoreActionCache.java b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/OnDiskBlobStoreActionCache.java
new file mode 100644
index 0000000..c038d2e
--- /dev/null
+++ b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/OnDiskBlobStoreActionCache.java
@@ -0,0 +1,33 @@
+// Copyright 2019 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.remote.worker;
+
+import build.bazel.remote.execution.v2.Digest;
+import com.google.devtools.build.lib.remote.SimpleBlobStoreActionCache;
+import com.google.devtools.build.lib.remote.disk.OnDiskBlobStore;
+import com.google.devtools.build.lib.remote.options.RemoteOptions;
+import com.google.devtools.build.lib.remote.util.DigestUtil;
+import com.google.devtools.build.lib.vfs.Path;
+
+/** A {@link SimpleBlobStoreActionCache} backed by an {@link OnDiskBlobStore}. */
+class OnDiskBlobStoreActionCache extends SimpleBlobStoreActionCache {
+
+ public OnDiskBlobStoreActionCache(RemoteOptions options, Path cacheDir, DigestUtil digestUtil) {
+ super(options, new OnDiskBlobStore(cacheDir), digestUtil);
+ }
+
+ public boolean containsKey(Digest digest) {
+ return ((OnDiskBlobStore) blobStore).contains(digest.getHash());
+ }
+}
diff --git a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java
index ab4396e..6abf699 100644
--- a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java
+++ b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java
@@ -31,9 +31,6 @@
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningScheduledExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
-import com.google.devtools.build.lib.remote.SimpleBlobStoreActionCache;
-import com.google.devtools.build.lib.remote.SimpleBlobStoreFactory;
-import com.google.devtools.build.lib.remote.common.SimpleBlobStore;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.remote.util.DigestUtil;
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
@@ -114,7 +111,7 @@
public RemoteWorker(
FileSystem fs,
RemoteWorkerOptions workerOptions,
- SimpleBlobStoreActionCache cache,
+ OnDiskBlobStoreActionCache cache,
Path sandboxPath,
DigestUtil digestUtil)
throws IOException {
@@ -246,34 +243,22 @@
sandboxPath = prepareSandboxRunner(fs, remoteWorkerOptions);
}
- logger.info("Initializing in-memory cache server.");
- boolean usingRemoteCache = SimpleBlobStoreFactory.isRemoteCacheOptions(remoteOptions);
- if (!usingRemoteCache) {
- logger.warning("Not using remote cache. This should be used for testing only!");
- }
- if ((remoteWorkerOptions.casPath != null)
- && (!PathFragment.create(remoteWorkerOptions.casPath).isAbsolute()
+ if (remoteWorkerOptions.casPath == null
+ || (!PathFragment.create(remoteWorkerOptions.casPath).isAbsolute()
|| !fs.getPath(remoteWorkerOptions.casPath).exists())) {
- logger.severe("--cas_path must refer to an existing, absolute path!");
+ logger.severe("--cas_path must be specified and refer to an exiting absolut path.");
System.exit(1);
return;
}
Path casPath =
remoteWorkerOptions.casPath != null ? fs.getPath(remoteWorkerOptions.casPath) : null;
- final SimpleBlobStore blobStore = SimpleBlobStoreFactory.create(remoteOptions, casPath);
-
+ DigestUtil digestUtil = new DigestUtil(fs.getDigestFunction());
+ OnDiskBlobStoreActionCache cache =
+ new OnDiskBlobStoreActionCache(remoteOptions, casPath, digestUtil);
ListeningScheduledExecutorService retryService =
MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1));
-
- DigestUtil digestUtil = new DigestUtil(fs.getDigestFunction());
- RemoteWorker worker =
- new RemoteWorker(
- fs,
- remoteWorkerOptions,
- new SimpleBlobStoreActionCache(remoteOptions, blobStore, digestUtil),
- sandboxPath,
- digestUtil);
+ RemoteWorker worker = new RemoteWorker(fs, remoteWorkerOptions, cache, sandboxPath, digestUtil);
final Server server = worker.startServer();
