src/main/java/com/google/devtools/build/lib/sandbox/AsynchronousTreeDeleter.java

// Copyright 2019 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.sandbox;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.flogger.GoogleLogger;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.devtools.build.lib.exec.TreeDeleter;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.vfs.Path;
import java.io.IOException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nullable;

/**
 * Executes file system tree deletions asynchronously.
 *
 * <p>The number of threads used to process the backlog of tree deletions can be configured at any
 * time via {@link #setThreads(int)}. While a build is running, this number should be low to not use
 * precious resources that could otherwise be used for the build itself. But when the build is
 * finished, this number should be raised to quickly go through any pending deletions.
 *
 * <p>The backlog is bounded: once {@link #MAX_PENDING_DELETIONS} trees are waiting, the caller
 * deletes its own tree synchronously until the backlog drains.
 */
public class AsynchronousTreeDeleter implements TreeDeleter {

  public static final String MOVED_TRASH_DIR = "_moved_trash_dir";

  /** Number of trees that may wait for deletion before callers delete their own. */
  static final int MAX_PENDING_DELETIONS = 256;

  private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();

  private final AtomicInteger trashCount = new AtomicInteger(0);

  /** Thread pool used to execute asynchronous tree deletions; null in synchronous mode. */
  @Nullable private ThreadPoolExecutor service;

  private final Path trashBase;

  private boolean trashBaseCreated = false;

  /** Constructs a new asynchronous tree deleter backed by just one thread. */
  public AsynchronousTreeDeleter(Path trashBase) {
    this(trashBase, MAX_PENDING_DELETIONS);
  }

  @VisibleForTesting
  AsynchronousTreeDeleter(Path trashBase, int maxPendingDeletions) {
    logger.atInfo().log("Starting async tree deletion pool with 1 thread");

    ThreadFactory threadFactory =
        new ThreadFactoryBuilder()
            .setNameFormat("tree-deleter")
            .setDaemon(true)
            .setPriority(Thread.MIN_PRIORITY)
            .build();

    // A single thread can fall behind a build indefinitely, and every pending tree keeps its
    // disk space, inodes and hardlinks to sandbox inputs. Once the queue is full, run the deletion
    // on the caller instead of queueing it. Unlike CallerRunsPolicy, this also runs it after
    // shutdown, so a deletion racing with shutdown is not dropped.
    service =
        new ThreadPoolExecutor(
            1,
            1,
            10L,
            TimeUnit.SECONDS,
            new ArrayBlockingQueue<>(maxPendingDeletions),
            threadFactory,
            (deletion, executor) -> deletion.run());

    this.trashBase = trashBase;
  }

  /**
   * Resizes the thread pool to the given number of threads.
   *
   * <p>If the pool of active threads is larger than the requested number of threads, the resize
   * will progressively happen as those active threads become inactive.
   *
   * @param threads desired number of threads
   */
  void setThreads(int threads) {
    checkState(threads > 0, "Use SynchronousTreeDeleter if no async behavior is desired");
    logger.atInfo().log("Resizing async tree deletion pool to %d threads", threads);
    ThreadPoolExecutor pool = checkNotNull(service, "Cannot call setThreads after shutdown");
    // Raise maximumPoolSize before corePoolSize when expanding to maintain the invariant
    // corePoolSize <= maximumPoolSize. When shrinking, only lower corePoolSize so existing
    // workers remain alive to drain remaining queued tasks in parallel before timing out.
    if (threads > pool.getMaximumPoolSize()) {
      pool.setMaximumPoolSize(threads);
    }
    pool.setCorePoolSize(threads);
  }

  @Override
  public void deleteTree(Path path) throws IOException {
    if (!path.exists()) {
      return;
    }
    if (path.getFileSystem() != trashBase.getFileSystem()) {
      path.deleteTree();
      return;
    }
    if (!trashBaseCreated) {
      trashBase.createDirectoryAndParents();
      trashBaseCreated = true;
    }
    Path trashPath = trashBase.getRelative(Integer.toString(trashCount.getAndIncrement()));
    try {
      path.renameTo(trashPath);
    } catch (IOException | IllegalArgumentException e) {
      logger.atWarning().withCause(e).log(
          "Failed to rename %s -> %s for asynchronous removal. Removing synchronously.",
          path, trashPath);
      path.deleteTree();
      return;
    }
    checkNotNull(service, "Cannot call deleteTree after shutdown")
        .execute(
            () -> {
              try (SilentCloseable c = Profiler.instance().profile("trashPath.deleteTree")) {
                trashPath.deleteTree();
              } catch (IOException e) {
                logger.atWarning().withCause(e).log(
                    "Failed to delete tree %s (originally %s) asynchronously", trashPath, path);
              }
            });
  }

  @Override
  public void shutdown() {
    if (service != null) {
      logger.atInfo().log("Finishing %d pending async tree deletions", service.getTaskCount());
      service.shutdown();
      try {
        service.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
      } catch (InterruptedException e) {
        logger.atWarning().withCause(e).log(
            "Interrupted while waiting for async tree deletions to finish");
        Thread.currentThread().interrupt();
      }
      service = null;
    }
  }

  public Path getTrashBase() {
    return trashBase;
  }
}