src/main/java/com/google/devtools/build/lib/server/IdleServerTasks.java

// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.server;

import com.google.devtools.build.lib.profiler.AutoProfiler;
import com.google.devtools.build.lib.util.LoggingUtil;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.util.ProcMeminfoParser;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.Symlinks;

import java.io.IOException;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.annotation.Nullable;

/**
 * Run cleanup-related tasks during idle periods in the server.
 * idle() and busy() must be called in that order, and only once.
 */
class IdleServerTasks {

  private final Path workspaceDir;
  private final ScheduledThreadPoolExecutor executor;
  private static final Logger LOG = Logger.getLogger(IdleServerTasks.class.getName());

  private static final long FIVE_MIN_MILLIS = 1000 * 60 * 5;

  /**
   * Must be called from the main thread.
   */
  public IdleServerTasks(@Nullable Path workspaceDir) {
    this.executor = new ScheduledThreadPoolExecutor(1);
    this.workspaceDir = workspaceDir;
  }

  /**
   * Called when the server becomes idle. Should not block, but may invoke
   * new threads.
   */
  public void idle() {
    Preconditions.checkState(!executor.isShutdown());

    // Do a GC cycle while the server is idle.
    executor.schedule(new Runnable() {
        @Override public void run() {
          try (AutoProfiler p = AutoProfiler.logged("Idle GC", LOG)) {
            System.gc();
          }
        }
      }, 10, TimeUnit.SECONDS);
  }

  /**
   * Called by the main thread when the server gets to work.
   * Should return quickly.
   */
  public void busy() {
    Preconditions.checkState(!executor.isShutdown());

    // Make sure tasks are finished after shutdown(), so they do not intefere
    // with subsequent server invocations.
    executor.shutdown();
    executor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
    executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);

    boolean interrupted = false;
    while (true) {
      try {
        executor.awaitTermination(Long.MAX_VALUE, TimeUnit.HOURS);
        break;
      } catch (InterruptedException e) {
        // It's unsafe to leak threads - just reset the interrupt bit later.
        interrupted = true;
      }
    }

    if (interrupted) {
      Thread.currentThread().interrupt();
    }
  }

  /**
   * Return true iff the server should continue processing requests.
   * Called from the main thread, so it should return quickly.
   */
  public boolean continueProcessing(long idleMillis) {
    if (!memoryHeuristic(idleMillis)) {
      return false;
    }
    if (workspaceDir == null) {
      return false;
    }

    FileStatus stat;
    try {
      stat = workspaceDir.statIfFound(Symlinks.FOLLOW);
    } catch (IOException e) {
      // Do not terminate the server if the workspace is temporarily inaccessible, for example,
      // if it is on a network filesystem and the connection is down.
      return true;
    }
    return stat != null && stat.isDirectory();
  }

  private boolean memoryHeuristic(long idleMillis) {
    if (idleMillis < FIVE_MIN_MILLIS) {
      // Don't check memory health until after five minutes.
      return true;
    }

    ProcMeminfoParser memInfo = null;
    try {
      memInfo = new ProcMeminfoParser();
    } catch (IOException e) {
      LOG.info("Could not process /proc/meminfo: " + e);
      return true;
    }

    long totalPhysical, totalFree;
    try {
      totalPhysical = memInfo.getTotalKb();
      totalFree = memInfo.getFreeRamKb(); // See method javadoc.
    } catch (IllegalArgumentException e) {
      // Ugly capture of unchecked exception, similar to that in
      // LocalHostCapacity.
      LoggingUtil.logToRemote(Level.WARNING,
          "Could not read memInfo during idle query", e);
      return true;
    }
    double fractionFree = (double) totalFree / totalPhysical;

    // If the system as a whole is low on memory, let this server die.
    if (fractionFree < .1) {
      LOG.info("Terminating due to memory constraints");
      LOG.info(String.format("Total physical:%d\nTotal free: %d\n",
                                         totalPhysical, totalFree));
      return false;
    }

    return true;
  }
}