when writing to local disk cache, open files later in order to avoid "too many open files"
Re-use the existing "LazyFileOutputStream" in DiskAndRemoteCacheClient.java
in order to avoid "Too many open files".
Resolves https://github.com/bazelbuild/bazel/issues/13435
Closes #13574.
PiperOrigin-RevId: 379892227
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java
index 150f618..2ac8029 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java
@@ -56,6 +56,7 @@
import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.DirectoryMetadata;
import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.FileMetadata;
import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.SymlinkMetadata;
+import com.google.devtools.build.lib.remote.common.LazyFileOutputStream;
import com.google.devtools.build.lib.remote.common.OutputDigestMismatchException;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.common.RemoteActionFileArtifactValue;
@@ -1099,57 +1100,6 @@
.build();
}
- /**
- * Creates an {@link OutputStream} that isn't actually opened until the first data is written.
- * This is useful to only have as many open file descriptors as necessary at a time to avoid
- * running into system limits.
- */
- private static class LazyFileOutputStream extends OutputStream {
-
- private final Path path;
- private OutputStream out;
-
- public LazyFileOutputStream(Path path) {
- this.path = path;
- }
-
- @Override
- public void write(byte[] b) throws IOException {
- ensureOpen();
- out.write(b);
- }
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- ensureOpen();
- out.write(b, off, len);
- }
-
- @Override
- public void write(int b) throws IOException {
- ensureOpen();
- out.write(b);
- }
-
- @Override
- public void flush() throws IOException {
- ensureOpen();
- out.flush();
- }
-
- @Override
- public void close() throws IOException {
- ensureOpen();
- out.close();
- }
-
- private void ensureOpen() throws IOException {
- if (out == null) {
- out = path.getOutputStream();
- }
- }
- }
-
/** In-memory representation of action result metadata. */
static class ActionResultMetadata {
diff --git a/src/main/java/com/google/devtools/build/lib/remote/common/LazyFileOutputStream.java b/src/main/java/com/google/devtools/build/lib/remote/common/LazyFileOutputStream.java
new file mode 100644
index 0000000..dbe0026
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/remote/common/LazyFileOutputStream.java
@@ -0,0 +1,69 @@
+// Copyright 2021 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.remote.common;
+
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Creates an {@link OutputStream} that isn't actually opened until the first data is written. This
+ * is useful to only have as many open file descriptors as necessary at a time to avoid running into
+ * system limits.
+ */
+public class LazyFileOutputStream extends OutputStream {
+
+ private final Path path;
+ private OutputStream out;
+
+ public LazyFileOutputStream(Path path) {
+ this.path = path;
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ ensureOpen();
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ ensureOpen();
+ out.write(b, off, len);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ ensureOpen();
+ out.write(b);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ ensureOpen();
+ out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ ensureOpen();
+ out.close();
+ }
+
+ private void ensureOpen() throws IOException {
+ if (out == null) {
+ out = path.getOutputStream();
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/remote/disk/DiskAndRemoteCacheClient.java b/src/main/java/com/google/devtools/build/lib/remote/disk/DiskAndRemoteCacheClient.java
index b340985..97a973e 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/disk/DiskAndRemoteCacheClient.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/disk/DiskAndRemoteCacheClient.java
@@ -20,6 +20,7 @@
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
+import com.google.devtools.build.lib.remote.common.LazyFileOutputStream;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
@@ -142,11 +143,7 @@
Path tempPath = newTempPath();
final OutputStream tempOut;
- try {
- tempOut = tempPath.getOutputStream();
- } catch (IOException e) {
- return Futures.immediateFailedFuture(e);
- }
+ tempOut = new LazyFileOutputStream(tempPath);
if (!options.incompatibleRemoteResultsIgnoreDisk || options.remoteAcceptCached) {
ListenableFuture<Void> download =
