src/main/java/com/google/devtools/build/lib/remote/disk/DiskAndRemoteCacheClient.java

// Copyright 2019 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.remote.disk;

import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
import static com.google.common.util.concurrent.Futures.immediateFuture;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;

import build.bazel.remote.execution.v2.ActionResult;
import build.bazel.remote.execution.v2.Digest;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.devtools.build.lib.remote.common.LazyFileOutputStream;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
import com.google.devtools.build.lib.vfs.Path;
import com.google.protobuf.ByteString;
import java.io.IOException;
import java.io.OutputStream;
import java.util.UUID;

/**
 * A {@link RemoteCacheClient} implementation combining two blob stores. A local disk blob store and
 * a remote blob store. If a blob isn't found in the first store, the second store is used, and the
 * blob added to the first. Put puts the blob on both stores.
 */
public final class DiskAndRemoteCacheClient implements RemoteCacheClient {

  private final RemoteCacheClient remoteCache;
  private final DiskCacheClient diskCache;

  public DiskAndRemoteCacheClient(DiskCacheClient diskCache, RemoteCacheClient remoteCache) {
    this.diskCache = Preconditions.checkNotNull(diskCache);
    this.remoteCache = Preconditions.checkNotNull(remoteCache);
  }

  @Override
  public ListenableFuture<Void> uploadActionResult(
      RemoteActionExecutionContext context, ActionKey actionKey, ActionResult actionResult) {
    ListenableFuture<Void> future = Futures.immediateVoidFuture();

    if (context.getWriteCachePolicy().allowDiskCache()) {
      future = diskCache.uploadActionResult(context, actionKey, actionResult);
    }

    if (context.getWriteCachePolicy().allowRemoteCache()) {
      future =
          Futures.transformAsync(
              future,
              v -> remoteCache.uploadActionResult(context, actionKey, actionResult),
              directExecutor());
    }
    return future;
  }

  @Override
  public void close() {
    diskCache.close();
    remoteCache.close();
  }

  @Override
  public ListenableFuture<Void> uploadFile(
      RemoteActionExecutionContext context, Digest digest, Path file) {
    ListenableFuture<Void> future = Futures.immediateVoidFuture();

    if (context.getWriteCachePolicy().allowDiskCache()) {
      future = diskCache.uploadFile(context, digest, file);
    }

    if (context.getWriteCachePolicy().allowRemoteCache()) {
      future =
          Futures.transformAsync(
              future, v -> remoteCache.uploadFile(context, digest, file), directExecutor());
    }
    return future;
  }

  @Override
  public ListenableFuture<Void> uploadBlob(
      RemoteActionExecutionContext context, Digest digest, ByteString data) {
    ListenableFuture<Void> future = Futures.immediateVoidFuture();

    if (context.getWriteCachePolicy().allowDiskCache()) {
      future = diskCache.uploadBlob(context, digest, data);
    }

    if (context.getWriteCachePolicy().allowRemoteCache()) {
      future =
          Futures.transformAsync(
              future, v -> remoteCache.uploadBlob(context, digest, data), directExecutor());
    }
    return future;
  }

  @Override
  public ListenableFuture<ImmutableSet<Digest>> findMissingDigests(
      RemoteActionExecutionContext context, Iterable<Digest> digests) {
    ListenableFuture<ImmutableSet<Digest>> diskQuery = immediateFuture(ImmutableSet.of());
    if (context.getWriteCachePolicy().allowDiskCache()) {
      diskQuery = diskCache.findMissingDigests(context, digests);
    }

    ListenableFuture<ImmutableSet<Digest>> remoteQuery = immediateFuture(ImmutableSet.of());
    if (context.getWriteCachePolicy().allowRemoteCache()) {
      remoteQuery = remoteCache.findMissingDigests(context, digests);
    }

    ListenableFuture<ImmutableSet<Digest>> diskQueryFinal = diskQuery;
    ListenableFuture<ImmutableSet<Digest>> remoteQueryFinal = remoteQuery;

    return Futures.whenAllSucceed(remoteQueryFinal, diskQueryFinal)
        .call(
            () ->
                ImmutableSet.<Digest>builder()
                    .addAll(remoteQueryFinal.get())
                    .addAll(diskQueryFinal.get())
                    .build(),
            directExecutor());
  }

  private Path newTempPath() {
    return diskCache.toPathNoSplit(UUID.randomUUID().toString());
  }

  private static ListenableFuture<Void> closeStreamOnError(
      ListenableFuture<Void> f, OutputStream out) {
    return Futures.catchingAsync(
        f,
        Exception.class,
        (rootCause) -> {
          try {
            out.close();
          } catch (IOException e) {
            rootCause.addSuppressed(e);
          }
          return immediateFailedFuture(rootCause);
        },
        directExecutor());
  }

  @Override
  public ListenableFuture<Void> downloadBlob(
      RemoteActionExecutionContext context, Digest digest, OutputStream out) {
    if (context.getReadCachePolicy().allowDiskCache() && diskCache.contains(digest)) {
      return diskCache.downloadBlob(context, digest, out);
    }

    Path tempPath = newTempPath();
    final OutputStream tempOut;
    tempOut = new LazyFileOutputStream(tempPath);

    if (context.getReadCachePolicy().allowRemoteCache()) {
      ListenableFuture<Void> download =
          closeStreamOnError(remoteCache.downloadBlob(context, digest, tempOut), tempOut);
      return Futures.transformAsync(
          download,
          (unused) -> {
            try {
              tempOut.close();
              diskCache.captureFile(tempPath, digest, /* isActionCache= */ false);
            } catch (IOException e) {
              return immediateFailedFuture(e);
            }
            return diskCache.downloadBlob(context, digest, out);
          },
          directExecutor());
    } else {
      return immediateFuture(null);
    }
  }

  @Override
  public ListenableFuture<CachedActionResult> downloadActionResult(
      RemoteActionExecutionContext context, ActionKey actionKey, boolean inlineOutErr) {
    if (context.getReadCachePolicy().allowDiskCache()
        && diskCache.containsActionResult(actionKey)) {
      return diskCache.downloadActionResult(context, actionKey, inlineOutErr);
    }

    if (context.getReadCachePolicy().allowRemoteCache()) {
      return Futures.transformAsync(
          remoteCache.downloadActionResult(context, actionKey, inlineOutErr),
          (cachedActionResult) -> {
            if (cachedActionResult == null) {
              return immediateFuture(null);
            } else {
              return Futures.transform(
                  diskCache.uploadActionResult(
                      context, actionKey, cachedActionResult.actionResult()),
                  v -> cachedActionResult,
                  directExecutor());
            }
          },
          directExecutor());
    } else {
      return immediateFuture(null);
    }
  }
}