Add --local_test_jobs to limit local test concurrency

Users have asked for ways to control the concurrency level of their
local tests. They can do it right now using the --local_resources
option, but that is unintuitive and affects the parallelism of
non-test actions.

This option changes the kind of resources obtained for local tests. If
set, the CPU, RAM, and IO dimensions for local tests will not be used,
and a new localTestCount dimension will be used, where the capacity is
equal to the option's value, and each local test consumes one unit.

--
MOS_MIGRATED_REVID=87177698
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
index 7f375d4..7255b5d 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
@@ -126,7 +126,8 @@
   static boolean isDisabled;
 
   // If /proc/* information is not available, assume 3000 MB and 2 CPUs.
-  private static ResourceSet DEFAULT_RESOURCES = ResourceSet.createWithRamCpuIo(3000.0, 2.0, 1.0);
+  private static ResourceSet DEFAULT_RESOURCES = ResourceSet.create(3000.0, 2.0, 1.0,
+      Integer.MAX_VALUE);
 
   private LocalHostCapacity() {}
 
@@ -252,10 +253,11 @@
       boolean hyperthreading = (logicalCpuCount != totalCores);
       double ramMb = ProcMeminfoParser.kbToMb(memInfo.getTotalKb());
       final double EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU = 0.6;
-      return ResourceSet.createWithRamCpuIo(
+      return ResourceSet.create(
           ramMb,
           logicalCpuCount * (hyperthreading ? EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU : 1.0),
-          1.0);
+          1.0,
+          Integer.MAX_VALUE);
     } catch (IOException | IllegalArgumentException e) {
       disableProcFsUse(e);
       return DEFAULT_RESOURCES;
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
index bb2aea9..db6ed11 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
@@ -33,14 +33,14 @@
 import java.util.logging.Logger;
 
 /**
- * CPU/RAM resource manager. Used to keep track of resources consumed by the Blaze action execution
- * threads and throttle them when necessary.
+ * Used to keep track of resources consumed by the Blaze action execution threads and throttle them
+ * when necessary.
  *
- * <p>Threads which are known to consume a significant amount of the local CPU or RAM resources
- * should call {@link #acquireResources} method. This method will check whether requested resources
- * are available and will either mark them as used and allow thread to proceed or will block the
- * thread until requested resources will become available. When thread completes it task, it must
- * release allocated resources by calling {@link #releaseResources} method.
+ * <p>Threads which are known to consume a significant amount of resources should call
+ * {@link #acquireResources} method. This method will check whether requested resources are
+ * available and will either mark them as used and allow the thread to proceed or will block the
+ * thread until requested resources will become available. When the thread completes its task, it
+ * must release allocated resources by calling {@link #releaseResources} method.
  *
  * <p>Available resources can be calculated using one of three ways:
  * <ol>
@@ -100,9 +100,9 @@
   // Please note that this value is purely empirical - we assume that generally
   // requested resources are somewhat pessimistic and thread would end up
   // using less than requested amount.
-  private final static double MIN_NECESSARY_CPU_RATIO = 0.6;
-  private final static double MIN_NECESSARY_RAM_RATIO = 1.0;
-  private final static double MIN_NECESSARY_IO_RATIO = 1.0;
+  private static final double MIN_NECESSARY_CPU_RATIO = 0.6;
+  private static final double MIN_NECESSARY_RAM_RATIO = 1.0;
+  private static final double MIN_NECESSARY_IO_RATIO = 1.0;
 
   // List of blocked threads. Associated CountDownLatch object will always
   // be initialized to 1 during creation in the acquire() method.
@@ -129,6 +129,9 @@
   // definition in the ResourceSet class.
   private double usedIo;
 
+  // Used local test count. Corresponds to the local test count definition in the ResourceSet class.
+  private int usedLocalTestCount;
+
   // Specifies how much of the RAM in staticResources we should allow to be used.
   public static final int DEFAULT_RAM_UTILIZATION_PERCENTAGE = 67;
   private int ramUtilizationPercentage = DEFAULT_RAM_UTILIZATION_PERCENTAGE;
@@ -138,7 +141,7 @@
 
   private ResourceManager() {
     FINE = LOG.isLoggable(Level.FINE);
-    requestList = new LinkedList<Pair<ResourceSet, CountDownLatch>>();
+    requestList = new LinkedList<>();
   }
 
   @VisibleForTesting public static ResourceManager instanceForTestingOnly() {
@@ -154,6 +157,7 @@
     usedCpu = 0;
     usedRam = 0;
     usedIo = 0;
+    usedLocalTestCount = 0;
     for (Pair<ResourceSet, CountDownLatch> request : requestList) {
       // CountDownLatch can be set only to 0 or 1.
       request.second.countDown();
@@ -220,7 +224,7 @@
    */
   public void acquireResources(ActionMetadata owner, ResourceSet resources)
       throws InterruptedException {
-    Preconditions.checkArgument(resources != null);
+    Preconditions.checkNotNull(resources);
     long startTime = Profiler.nanoTimeMaybe();
     CountDownLatch latch = null;
     try {
@@ -231,7 +235,7 @@
       }
     } finally {
       threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0
-          || resources.getIoUsage() != 0);
+          || resources.getIoUsage() != 0 || resources.getLocalTestCount() != 0);
       acquired(owner);
 
       // Profile acquisition only if it waited for resource to become available.
@@ -255,7 +259,8 @@
     }
 
     if (acquired) {
-      threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0);
+      threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0
+          || resources.getIoUsage() != 0 || resources.getLocalTestCount() != 0);
       acquired(owner);
     }
 
@@ -266,13 +271,15 @@
     usedCpu += resources.getCpuUsage();
     usedRam += resources.getMemoryMb();
     usedIo += resources.getIoUsage();
+    usedLocalTestCount += resources.getLocalTestCount();
   }
 
   /**
    * Return true if any resources have been claimed through this manager.
    */
   public synchronized boolean inUse() {
-    return usedCpu != 0.0 || usedRam != 0.0 || usedIo != 0.0 || requestList.size() > 0;
+    return usedCpu != 0.0 || usedRam != 0.0 || usedIo != 0.0 || usedLocalTestCount != 0
+        || requestList.size() > 0;
   }
 
 
@@ -353,16 +360,18 @@
     usedCpu -= resources.getCpuUsage();
     usedRam -= resources.getMemoryMb();
     usedIo -= resources.getIoUsage();
+    usedLocalTestCount -= resources.getLocalTestCount();
 
     // TODO(bazel-team): (2010) rounding error can accumulate and value below can end up being
     // e.g. 1E-15. So if it is small enough, we set it to 0. But maybe there is a better solution.
-    if (usedCpu < 0.0001) {
+    double epsilon = 0.0001;
+    if (usedCpu < epsilon) {
       usedCpu = 0;
     }
-    if (usedRam < 0.0001) {
+    if (usedRam < epsilon) {
       usedRam = 0;
     }
-    if (usedIo < 0.0001) {
+    if (usedIo < epsilon) {
       usedIo = 0;
     }
     if (requestList.size() > 0) {
@@ -393,7 +402,7 @@
     Preconditions.checkNotNull(availableResources);
     // Comparison below is robust, since any calculation errors will be fixed
     // by the release() method.
-    if (usedCpu == 0.0 && usedRam == 0.0 && usedIo == 0.0) {
+    if (usedCpu == 0.0 && usedRam == 0.0 && usedIo == 0.0 && usedLocalTestCount == 0) {
       return true;
     }
     // Use only MIN_NECESSARY_???_RATIO of the resource value to check for
@@ -404,10 +413,12 @@
     double cpu = resources.getCpuUsage() * MIN_NECESSARY_CPU_RATIO;
     double ram = resources.getMemoryMb() * MIN_NECESSARY_RAM_RATIO;
     double io = resources.getIoUsage() * MIN_NECESSARY_IO_RATIO;
+    int localTestCount = resources.getLocalTestCount();
 
     double availableCpu = availableResources.getCpuUsage();
     double availableRam = availableResources.getMemoryMb();
     double availableIo = availableResources.getIoUsage();
+    int availableLocalTestCount = availableResources.getLocalTestCount();
 
     // Resources are considered available if any one of the conditions below is true:
     // 1) If resource is not requested at all, it is available.
@@ -416,28 +427,33 @@
     // ensure that at any given time, at least one thread is able to acquire
     // resources even if it requests more than available.
     // 3) If used resource amount is less than total available resource amount.
-    return (cpu == 0.0 || usedCpu == 0.0 || usedCpu + cpu <= availableCpu) &&
-        (ram == 0.0 || usedRam == 0.0 || usedRam + ram <= availableRam) &&
-        (io == 0.0 || usedIo == 0.0 || usedIo + io <= availableIo);
+    boolean cpuIsAvailable = cpu == 0.0 || usedCpu == 0.0 || usedCpu + cpu <= availableCpu;
+    boolean ramIsAvailable = ram == 0.0 || usedRam == 0.0 || usedRam + ram <= availableRam;
+    boolean ioIsAvailable = io == 0.0 || usedIo == 0.0 || usedIo + io <= availableIo;
+    boolean localTestCountIsAvailable = localTestCount == 0 || usedLocalTestCount == 0
+        || usedLocalTestCount + localTestCount <= availableLocalTestCount;
+    return cpuIsAvailable && ramIsAvailable && ioIsAvailable && localTestCountIsAvailable;
   }
 
   private synchronized void updateAvailableResources(boolean useFreeReading) {
     Preconditions.checkNotNull(staticResources);
     if (useFreeReading && isAutoSensingEnabled()) {
-      availableResources = ResourceSet.createWithRamCpuIo(
+      availableResources = ResourceSet.create(
           usedRam + freeReading.getFreeMb(),
           usedCpu + freeReading.getAvgFreeCpu(),
-          staticResources.getIoUsage());
+          staticResources.getIoUsage(),
+          staticResources.getLocalTestCount());
       if(FINE) {
         LOG.fine("Free resources: " + Math.round(freeReading.getFreeMb()) + " MB,"
             + Math.round(freeReading.getAvgFreeCpu() * 100) + "% CPU");
       }
       processWaitingThreads();
     } else {
-      availableResources = ResourceSet.createWithRamCpuIo(
+      availableResources = ResourceSet.create(
           staticResources.getMemoryMb() * this.ramUtilizationPercentage / 100.0,
           staticResources.getCpuUsage(),
-          staticResources.getIoUsage());
+          staticResources.getIoUsage(),
+          staticResources.getLocalTestCount());
       processWaitingThreads();
     }
   }
@@ -466,7 +482,7 @@
   }
 
   @VisibleForTesting
-  synchronized boolean isAvailable(double ram, double cpu, double io) {
-    return areResourcesAvailable(ResourceSet.createWithRamCpuIo(ram, cpu, io));
+  synchronized boolean isAvailable(double ram, double cpu, double io, int localTestCount) {
+    return areResourcesAvailable(ResourceSet.create(ram, cpu, io, localTestCount));
   }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java
index 65d2d14..f4f1327 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java
@@ -33,7 +33,7 @@
 public class ResourceSet {
 
   /** For actions that consume negligible resources. */
-  public static final ResourceSet ZERO = new ResourceSet(0.0, 0.0, 0.0);
+  public static final ResourceSet ZERO = new ResourceSet(0.0, 0.0, 0.0, 0);
 
   /** The amount of real memory (resident set size). */
   private final double memoryMb;
@@ -41,20 +41,49 @@
   /** The number of CPUs, or fractions thereof. */
   private final double cpuUsage;
 
+  /** The number of local tests. */
+  private final int localTestCount;
+
   /**
    * Relative amount of used I/O resources (with 1.0 being total available amount on an "average"
    * workstation.
    */
   private final double ioUsage;
   
-  private ResourceSet(double memoryMb, double cpuUsage, double ioUsage) {
+  private ResourceSet(double memoryMb, double cpuUsage, double ioUsage, int localTestCount) {
     this.memoryMb = memoryMb;
     this.cpuUsage = cpuUsage;
     this.ioUsage = ioUsage;
+    this.localTestCount = localTestCount;
   }
 
+  /**
+   * Returns a new ResourceSet with the provided values for memoryMb, cpuUsage, and ioUsage, and
+   * with 0.0 for localTestCount. Use this method in action resource definitions when they aren't
+   * local tests.
+   */
   public static ResourceSet createWithRamCpuIo(double memoryMb, double cpuUsage, double ioUsage) {
-    return new ResourceSet(memoryMb, cpuUsage, ioUsage);
+    return new ResourceSet(memoryMb, cpuUsage, ioUsage, 0);
+  }
+
+  /**
+   * Returns a new ResourceSet with the provided value for localTestCount, and 0.0 for memoryMb,
+   * cpuUsage, and ioUsage. Use this method in action resource definitions when they are local tests
+   * that acquire no local resources.
+   */
+  public static ResourceSet createWithLocalTestCount(int localTestCount) {
+    return new ResourceSet(0.0, 0.0, 0.0, localTestCount);
+  }
+
+  /**
+   * Returns a new ResourceSet with the provided values for memoryMb, cpuUsage, ioUsage, and
+   * localTestCount. Most action resource definitions should use
+   * {@link #createWithRamCpuIo(double, double, double)} or {@link #createWithLocalTestCount(int)}.
+   * Use this method primarily when constructing ResourceSets that represent available resources.
+   */
+  public static ResourceSet create(double memoryMb, double cpuUsage, double ioUsage,
+      int localTestCount) {
+    return new ResourceSet(memoryMb, cpuUsage, ioUsage, localTestCount);
   }
 
   /** Returns the amount of real memory (resident set size) used in MB. */
@@ -83,6 +112,11 @@
     return ioUsage;
   }
 
+  /** Returns the local test count used. */
+  public int getLocalTestCount() {
+    return localTestCount;
+  }
+
   public static class ResourceSetConverter implements Converter<ResourceSet> {
     private static final Splitter SPLITTER = Splitter.on(',');
 
@@ -99,11 +133,9 @@
         if (memoryMb <= 0.0 || cpuUsage <= 0.0 || ioUsage <= 0.0) {
           throw new OptionsParsingException("All resource values must be positive");
         }
-        return createWithRamCpuIo(memoryMb, cpuUsage, ioUsage);
-      } catch (NumberFormatException nfe) {
+        return create(memoryMb, cpuUsage, ioUsage, Integer.MAX_VALUE);
+      } catch (NumberFormatException | NoSuchElementException nfe) {
         throw new OptionsParsingException("Expected exactly 3 comma-separated float values", nfe);
-      } catch (NoSuchElementException nsee) {
-        throw new OptionsParsingException("Expected exactly 3 comma-separated float values", nsee);
       }
     }
 
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java
index 18a03ad..0b56fa0 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java
@@ -460,6 +460,13 @@
   }
 
   /**
+   * Returns the set of execution options specified for this request.
+   */
+  public ExecutionOptions getExecutionOptions() {
+    return getOptions(ExecutionOptions.class);
+  }
+
+  /**
    * Returns the human-readable description of the non-default options
    * for this build request.
    */
@@ -497,6 +504,18 @@
           String.format("High value for --jobs: %d. You may run into memory issues", jobs));
     }
 
+    int localTestJobs = getExecutionOptions().localTestJobs;
+    if (localTestJobs < 0) {
+      throw new InvalidConfigurationException(String.format(
+          "Invalid parameter for --local_test_jobs: %d. Only values 0 or greater are "
+              + "allowed.", localTestJobs));
+    }
+    if (localTestJobs > jobs) {
+      warnings.add(
+          String.format("High value for --local_test_jobs: %d. This exceeds the value for --jobs: "
+              + "%d. Only up to %d local tests will run concurrently.", localTestJobs, jobs, jobs));
+    }
+
     // Validate other BuildRequest options.
     if (getBuildOptions().verboseExplanations && getBuildOptions().explanationPath == null) {
       warnings.add("--verbose_explanations has no effect when --explain=<file> is not enabled");
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
index 0c8d404..b248b05 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -42,6 +42,7 @@
 import com.google.devtools.build.lib.actions.ExecutorInitException;
 import com.google.devtools.build.lib.actions.LocalHostCapacity;
 import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.ResourceSet;
 import com.google.devtools.build.lib.actions.SpawnActionContext;
 import com.google.devtools.build.lib.actions.TestExecException;
 import com.google.devtools.build.lib.actions.cache.ActionCache;
@@ -760,11 +761,12 @@
   private void configureResourceManager(BuildRequest request) {
     ResourceManager resourceMgr = ResourceManager.instance();
     ExecutionOptions options = request.getOptions(ExecutionOptions.class);
+    ResourceSet resources;
     if (options.availableResources != null) {
-      resourceMgr.setAvailableResources(options.availableResources);
+      resources = options.availableResources;
       resourceMgr.setRamUtilizationPercentage(100);
     } else {
-      resourceMgr.setAvailableResources(LocalHostCapacity.getLocalHostCapacity());
+      resources = LocalHostCapacity.getLocalHostCapacity();
       resourceMgr.setRamUtilizationPercentage(options.ramUtilizationPercentage);
       if (options.useResourceAutoSense) {
         getReporter().handle(
@@ -772,6 +774,14 @@
       }
       ResourceManager.instance().setAutoSensing(/*autosense=*/false);
     }
+
+    resourceMgr.setAvailableResources(ResourceSet.create(
+        resources.getMemoryMb(),
+        resources.getCpuUsage(),
+        resources.getIoUsage(),
+        request.getExecutionOptions().usingLocalTestJobs()
+            ? request.getExecutionOptions().localTestJobs : Integer.MAX_VALUE
+    ));
   }
 
   /**
diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
index 58e360b..a12ece9 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
@@ -192,4 +192,17 @@
       converter = ResourceSet.ResourceSetConverter.class
       )
   public ResourceSet availableResources;
+
+  @Option(name = "local_test_jobs",
+      defaultValue = "0",
+      category = "testing",
+      help = "The max number of local test jobs to run concurrently. "
+          + "0 means local resources will limit the number of local test jobs to run "
+          + "concurrently instead. Setting this greater than the value for --jobs is ineffectual."
+  )
+  public int localTestJobs;
+
+  public boolean usingLocalTestJobs() {
+    return localTestJobs != 0;
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java
index 72970a8..d0ff957 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java
@@ -86,7 +86,7 @@
     Spawn spawn = new BaseSpawn(getArgs(action), env,
         action.getTestProperties().getExecutionInfo(),
         action,
-        action.getTestProperties().getLocalResourceUsage());
+        action.getTestProperties().getLocalResourceUsage(executionOptions.usingLocalTestJobs()));
 
     Executor executor = actionExecutionContext.getExecutor();
 
@@ -97,7 +97,8 @@
       fileOutErr = new FileOutErr(action.getTestLog().getPath(),
           action.resolve(actionExecutionContext.getExecutor().getExecRoot()).getTestStderr());
 
-      resources = action.getTestProperties().getLocalResourceUsage();
+      resources = action.getTestProperties()
+          .getLocalResourceUsage(executionOptions.usingLocalTestJobs());
       ResourceManager.instance().acquireResources(action, resources);
       TestResultData data = execute(
           actionExecutionContext.withFileOutErr(fileOutErr), spawn, action);
diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java
index 0834ca9..0e0befc 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java
@@ -38,11 +38,12 @@
   /**
    * Resources used by local tests of various sizes.
    */
-  private static final ResourceSet SMALL_RESOURCES = ResourceSet.createWithRamCpuIo(20, 0.9, 0.00);
-  private static final ResourceSet MEDIUM_RESOURCES = ResourceSet.createWithRamCpuIo(100, 0.9, 0.1);
-  private static final ResourceSet LARGE_RESOURCES = ResourceSet.createWithRamCpuIo(300, 0.8, 0.1);
-  private static final ResourceSet ENORMOUS_RESOURCES =
-      ResourceSet.createWithRamCpuIo(800, 0.7, 0.4);
+  private static final ResourceSet SMALL_RESOURCES = ResourceSet.create(20, 0.9, 0.00, 1);
+  private static final ResourceSet MEDIUM_RESOURCES = ResourceSet.create(100, 0.9, 0.1, 1);
+  private static final ResourceSet LARGE_RESOURCES = ResourceSet.create(300, 0.8, 0.1, 1);
+  private static final ResourceSet ENORMOUS_RESOURCES = ResourceSet.create(800, 0.7, 0.4, 1);
+  private static final ResourceSet LOCAL_TEST_JOBS_BASED_RESOURCES =
+      ResourceSet.createWithLocalTestCount(1);
 
   private static ResourceSet getResourceSetFromSize(TestSize size) {
     switch (size) {
@@ -115,8 +116,10 @@
     return isExternal;
   }
 
-  public ResourceSet getLocalResourceUsage() {
-    return TestTargetProperties.getResourceSetFromSize(size);
+  public ResourceSet getLocalResourceUsage(boolean usingLocalTestJobs) {
+    return usingLocalTestJobs
+        ? LOCAL_TEST_JOBS_BASED_RESOURCES
+        : TestTargetProperties.getResourceSetFromSize(size);
   }
 
   /**
diff --git a/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java b/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java
index eebf0a9..c8e0af4 100644
--- a/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java
@@ -17,12 +17,10 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.eventbus.EventBus;
 import com.google.devtools.build.lib.testutil.TestThread;
-import com.google.devtools.common.options.OptionsParsingException;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -51,7 +49,9 @@
   @Before
   public void setUp() throws Exception {
     rm.setRamUtilizationPercentage(100);
-    rm.setAvailableResources(ResourceSet.createWithRamCpuIo(1000, 1, 1));
+    rm.setAvailableResources(
+        ResourceSet.create(/*memoryMb=*/1000.0, /*cpuUsage=*/1.0, /*ioUsage=*/1.0,
+        /*testCount=*/2));
     rm.setEventBus(new EventBus());
     counter = new AtomicInteger(0);
     sync = new CyclicBarrier(2);
@@ -59,16 +59,17 @@
     rm.resetResourceUsage();
   }
 
-  private void acquire(double ram, double cpu, double io) throws InterruptedException {
-    rm.acquireResources(resourceOwner, ResourceSet.createWithRamCpuIo(ram, cpu, io));
+  private void acquire(double ram, double cpu, double io, int tests)
+      throws InterruptedException {
+    rm.acquireResources(resourceOwner, ResourceSet.create(ram, cpu, io, tests));
   }
 
-  private boolean acquireNonblocking(double ram, double cpu, double io) {
-    return rm.tryAcquire(resourceOwner, ResourceSet.createWithRamCpuIo(ram, cpu, io));
+  private boolean acquireNonblocking(double ram, double cpu, double io, int tests) {
+    return rm.tryAcquire(resourceOwner, ResourceSet.create(ram, cpu, io, tests));
   }
 
-  private void release(double ram, double cpu, double io) {
-    rm.releaseResources(resourceOwner, ResourceSet.createWithRamCpuIo(ram, cpu, io));
+  private void release(double ram, double cpu, double io, int tests) {
+    rm.releaseResources(resourceOwner, ResourceSet.create(ram, cpu, io, tests));
   }
 
   private void validate (int count) {
@@ -76,57 +77,122 @@
   }
 
   @Test
-  public void testIndependentLargeRequests() throws Exception {
-    // Available: 1000 RAM and 1 CPU.
+  public void testOverBudgetRequests() throws Exception {
     assertFalse(rm.inUse());
-    acquire(10000, 0, 0); // Available: 0 RAM 1 CPU 1 IO.
-    acquire(0, 100, 0);   // Available: 0 RAM 0 CPU 1 IO.
-    acquire(0, 0, 1);     // Available: 0 RAM 0 CPU 0 IO.
+
+    // When nothing is consuming RAM,
+    // Then Resource Manager will successfully acquire an over-budget request for RAM:
+    double bigRam = 10000.0;
+    acquire(bigRam, 0, 0, 0);
+    // When RAM is consumed,
+    // Then Resource Manager will be "in use":
     assertTrue(rm.inUse());
-    release(9500, 0, 0);  // Available: 500 RAM 0 CPU 0 IO.
-    acquire(400, 0, 0);   // Available: 100 RAM 0 CPU 0 IO.
-    release(0, 99.5, 0.6);  // Available: 100 RAM 0.5 CPU 0.4 IO.
-    acquire(100, 0.5, 0.4); // Available: 0 RAM 0 CPU 0 IO.
-    release(1000, 1, 1);  // Available: 1000 RAM 1 CPU 1 IO.
+    release(bigRam, 0, 0, 0);
+    // When that RAM is released,
+    // Then Resource Manager will not be "in use":
+    assertFalse(rm.inUse());
+
+    // Ditto, for CPU:
+    double bigCpu = 10.0;
+    acquire(0, bigCpu, 0, 0);
+    assertTrue(rm.inUse());
+    release(0, bigCpu, 0, 0);
+    assertFalse(rm.inUse());
+
+    // Ditto, for IO:
+    double bigIo = 10.0;
+    acquire(0, 0, bigIo, 0);
+    assertTrue(rm.inUse());
+    release(0, 0, bigIo, 0);
+    assertFalse(rm.inUse());
+
+    // Ditto, for tests:
+    int bigTests = 10;
+    acquire(0, 0, 0, bigTests);
+    assertTrue(rm.inUse());
+    release(0, 0, 0, bigTests);
     assertFalse(rm.inUse());
   }
 
   @Test
-  public void testOverallocation() throws Exception {
-    // Since ResourceManager.MIN_NECESSARY_RAM_RATIO = 1.0, overallocation is
-    // enabled only for the CPU resource.
+  public void testThatCpuCanBeOverallocated() throws Exception {
     assertFalse(rm.inUse());
-    acquire(900, 0.5, 0.1);  // Available: 100 RAM 0.5 CPU 0.9 IO.
-    acquire(100, 0.6, 0.9);  // Available: 0 RAM 0 CPU 0 IO.
-    release(100, 0.6, 0.9);  // Available: 100 RAM 0.5 CPU 0.9 IO.
-    acquire(100, 0.1, 0.1);  // Available: 0 RAM 0.4 CPU 0.8 IO.
-    acquire(0, 0.5, 0.8);    // Available: 0 RAM 0 CPU 0.8 IO.
-    release(1020, 1.1, 1.05); // Available: 1000 RAM 1 CPU 1 IO.
-    assertFalse(rm.inUse());
+
+    // Given CPU is partially acquired:
+    acquire(0, 0.5, 0, 0);
+
+    // When a request for CPU is made that would slightly overallocate CPU,
+    // Then the request succeeds:
+    assertTrue(acquireNonblocking(0, 0.6, 0, 0));
   }
 
   @Test
-  public void testNonblocking() throws Exception {
+  public void testThatCpuAllocationIsNoncommutative() throws Exception {
     assertFalse(rm.inUse());
-    assertTrue(acquireNonblocking(900, 0.5, 0));  // Available: 100 RAM 0.5 CPU 1 IO.
-    assertTrue(acquireNonblocking(100, 0.5, 0.2));  // Available: 0 RAM 0 CPU 0.8 IO.
-    assertFalse(acquireNonblocking(.1, .01, 0.0));
-    assertFalse(acquireNonblocking(0, 0, 0.9));
-    assertTrue(acquireNonblocking(0, 0, 0.8));  // Available: 0 RAM 0 CPU 0 IO.
-    release(100, 0.5, 0.1);  // Available: 100 RAM 0.5 CPU 0.1 IO.
-    assertTrue(acquireNonblocking(100, 0.1, 0.1));  // Available: 0 RAM 0.4 CPU 0 IO.
-    assertFalse(acquireNonblocking(5, .5, 0));
-    assertFalse(acquireNonblocking(0, .5, 0.1));
-    assertTrue(acquireNonblocking(0, 0.4, 0));    // Available: 0 RAM 0 CPU 0 IO.
-    release(1000, 1, 1); // Available: 1000 RAM 1 CPU 1 IO.
+
+    // Given that CPU has a small initial allocation:
+    acquire(0, 0.099, 0, 0);
+
+    // When a request for a large CPU allocation is made,
+    // Then the request succeeds:
+    assertTrue(acquireNonblocking(0, 0.99, 0, 0));
+
+    // Cleanup
+    release(0, 1.089, 0, 0);
     assertFalse(rm.inUse());
+
+
+    // Given that CPU has a large initial allocation:
+    acquire(0, 0.99, 0, 0);
+
+    // When a request for a small CPU allocation is made,
+    // Then the request fails:
+    assertFalse(acquireNonblocking(0, 0.099, 0, 0));
+
+    // Note that this behavior is surprising and probably not intended.
+  }
+
+  @Test
+  public void testThatRamCannotBeOverallocated() throws Exception {
+    assertFalse(rm.inUse());
+
+    // Given RAM is partially acquired:
+    acquire(500, 0, 0, 0);
+
+    // When a request for RAM is made that would slightly overallocate RAM,
+    // Then the request fails:
+    assertFalse(acquireNonblocking(600, 0, 0, 0));
+  }
+
+  @Test
+  public void testThatIOCannotBeOverallocated() throws Exception {
+    assertFalse(rm.inUse());
+
+    // Given IO is partially acquired:
+    acquire(0, 0, 0.5, 0);
+
+    // When a request for IO is made that would slightly overallocate IO,
+    // Then the request fails:
+    assertFalse(acquireNonblocking(0, 0, 0.6, 0));
+  }
+
+  @Test
+  public void testThatTestsCannotBeOverallocated() throws Exception {
+    assertFalse(rm.inUse());
+
+    // Given test count is partially acquired:
+    acquire(0, 0, 0, 1);
+
+    // When a request for tests is made that would slightly overallocate tests,
+    // Then the request fails:
+    assertFalse(acquireNonblocking(0, 0, 0, 2));
   }
 
   @Test
   public void testHasResources() throws Exception {
     assertFalse(rm.inUse());
     assertFalse(rm.threadHasResources());
-    acquire(1, .1, .1);
+    acquire(1.0, 0.1, 0.1, 1);
     assertTrue(rm.threadHasResources());
 
     // We have resources in this thread - make sure other threads
@@ -134,24 +200,28 @@
     TestThread thread1 = new TestThread () {
       @Override public void runTest() throws Exception {
         assertFalse(rm.threadHasResources());
-        acquire(1, 0, 0);
+        acquire(1.0, 0, 0, 0);
         assertTrue(rm.threadHasResources());
-        release(1, 0, 0);
+        release(1.0, 0, 0, 0);
         assertFalse(rm.threadHasResources());
-        acquire(0, 0.1, 0);
+        acquire(0, 0.1, 0, 0);
         assertTrue(rm.threadHasResources());
-        release(0, 0.1, 0);
+        release(0, 0.1, 0, 0);
         assertFalse(rm.threadHasResources());
-        acquire(0, 0, 0.1);
+        acquire(0, 0, 0.1, 0);
         assertTrue(rm.threadHasResources());
-        release(0, 0, 0.1);
+        release(0, 0, 0.1, 0);
+        assertFalse(rm.threadHasResources());
+        acquire(0, 0, 0, 1);
+        assertTrue(rm.threadHasResources());
+        release(0, 0, 0, 1);
         assertFalse(rm.threadHasResources());
       }
     };
     thread1.start();
     thread1.joinAndAssertState(10000);
 
-    release(1, .1, .1);
+    release(1.0, 0.1, 0.1, 1);
     assertFalse(rm.threadHasResources());
     assertFalse(rm.inUse());
   }
@@ -161,7 +231,7 @@
     assertFalse(rm.inUse());
     TestThread thread1 = new TestThread () {
       @Override public void runTest() throws Exception {
-        acquire(2000, 2, 0);
+        acquire(2000, 2, 0, 0);
         sync.await();
         validate(1);
         sync.await();
@@ -169,25 +239,25 @@
         while (rm.getWaitCount() == 0) {
           Thread.yield();
         }
-        release(2000, 2, 0);
+        release(2000, 2, 0, 0);
         assertEquals(0, rm.getWaitCount());
-        acquire(2000, 2, 0); // Will be blocked by the thread2.
+        acquire(2000, 2, 0, 0); // Will be blocked by the thread2.
         validate(3);
-        release(2000, 2, 0);
+        release(2000, 2, 0, 0);
       }
     };
     TestThread thread2 = new TestThread () {
       @Override public void runTest() throws Exception {
         sync2.await();
-        assertFalse(rm.isAvailable(2000, 2, 0));
-        acquire(2000, 2, 0); // Will be blocked by the thread1.
+        assertFalse(rm.isAvailable(2000, 2, 0, 0));
+        acquire(2000, 2, 0, 0); // Will be blocked by the thread1.
         validate(2);
         sync2.await();
         // Wait till other thread will be locked.
         while (rm.getWaitCount() == 0) {
           Thread.yield();
         }
-        release(2000, 2, 0);
+        release(2000, 2, 0, 0);
       }
     };
 
@@ -210,9 +280,9 @@
     TestThread thread1 = new TestThread () {
       @Override public void runTest() throws Exception {
         sync.await();
-        acquire(900, 0.5, 0); // Will be blocked by the main thread.
+        acquire(900, 0.5, 0, 0); // Will be blocked by the main thread.
         validate(5);
-        release(900, 0.5, 0);
+        release(900, 0.5, 0, 0);
         sync.await();
       }
     };
@@ -222,17 +292,17 @@
         while (rm.getWaitCount() == 0) {
           Thread.yield();
         }
-        acquire(100, 0.1, 0);
+        acquire(100, 0.1, 0, 0);
         validate(2);
-        release(100, 0.1, 0);
+        release(100, 0.1, 0, 0);
         sync2.await();
-        acquire(200, 0.5, 0);
+        acquire(200, 0.5, 0, 0);
         validate(4);
         sync2.await();
-        release(200, 0.5, 0);
+        release(200, 0.5, 0, 0);
       }
     };
-    acquire(900, 0.9, 0);
+    acquire(900, 0.9, 0, 0);
     validate(1);
     thread1.start();
     sync.await(1, TimeUnit.SECONDS);
@@ -243,20 +313,16 @@
       Thread.yield();
     }
     validate(3); // Thread1 is now first in the queue and Thread2 is second.
-    release(100, 0.4, 0); // This allows Thread2 to continue out of order.
+    release(100, 0.4, 0, 0); // This allows Thread2 to continue out of order.
     sync2.await(1, TimeUnit.SECONDS);
-    release(750, 0.3, 0); // At this point thread1 will finally acquire resources.
+    release(750, 0.3, 0, 0); // At this point thread1 will finally acquire resources.
     sync.await(1, TimeUnit.SECONDS);
-    release(50, 0.2, 0);
+    release(50, 0.2, 0, 0);
     thread1.join();
     thread2.join();
     assertFalse(rm.inUse());
   }
 
-  @Test
-  public void testSingleton() throws Exception {
-    ResourceManager.instance();
-  }
 
   /**
    * Checks that that resource manager
@@ -287,37 +353,6 @@
     assertFalse(rm.inUse());
   }
 
-  @Test
-  public void testResourceSetConverter() throws Exception {
-    ResourceSet.ResourceSetConverter converter = new ResourceSet.ResourceSetConverter();
-
-    ResourceSet resources = converter.convert("1,0.5,2");
-    assertEquals(1.0, resources.getMemoryMb(), 0.01);
-    assertEquals(0.5, resources.getCpuUsage(), 0.01);
-    assertEquals(2.0, resources.getIoUsage(), 0.01);
-
-    try {
-      converter.convert("0,0,");
-      fail();
-    } catch (OptionsParsingException ope) {
-      // expected
-    }
-
-    try {
-      converter.convert("0,0,0,0");
-      fail();
-    } catch (OptionsParsingException ope) {
-      // expected
-    }
-
-    try {
-      converter.convert("-1,0,0");
-      fail();
-    } catch (OptionsParsingException ope) {
-      // expected
-    }
-  }
-
   private static class ResourceOwnerStub implements ActionMetadata {
 
     @Override
diff --git a/src/test/java/com/google/devtools/build/lib/actions/ResourceSetTest.java b/src/test/java/com/google/devtools/build/lib/actions/ResourceSetTest.java
new file mode 100644
index 0000000..0f88b69
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/ResourceSetTest.java
@@ -0,0 +1,67 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.actions;
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import com.google.devtools.build.lib.actions.ResourceSet.ResourceSetConverter;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Tests for @{link ResourceSet}.
+ */
+@RunWith(JUnit4.class)
+public class ResourceSetTest {
+
+  private ResourceSetConverter converter;
+
+  @Before
+  public void setUp() throws Exception {
+    converter = new ResourceSetConverter();
+  }
+
+  @Test
+  public void testConverterParsesExpectedFormat() throws Exception {
+    ResourceSet resources = converter.convert("1,0.5,2");
+    assertEquals(1.0, resources.getMemoryMb(), 0.01);
+    assertEquals(0.5, resources.getCpuUsage(), 0.01);
+    assertEquals(2.0, resources.getIoUsage(), 0.01);
+    assertEquals(Integer.MAX_VALUE, resources.getLocalTestCount());
+  }
+
+  @Test(expected = OptionsParsingException.class)
+  public void testConverterThrowsWhenGivenInsufficientInputs() throws Exception {
+    converter.convert("0,0,");
+    fail();
+  }
+
+  @Test(expected = OptionsParsingException.class)
+  public void testConverterThrowsWhenGivenTooManyInputs() throws Exception {
+    converter.convert("0,0,0,");
+    fail();
+  }
+
+  @Test(expected = OptionsParsingException.class)
+  public void testConverterThrowsWhenGivenNegativeInputs() throws Exception {
+    converter.convert("-1,0,0");
+    fail();
+  }
+}