Collect action cache hits, misses, and reasons for the misses.
As a bonus, this brings in a bunch of new unit tests for the
ActionCacheChecker.
RELNOTES: None.
PiperOrigin-RevId: 170059577
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java
index 1a6c4c8..fc64c75 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java
@@ -24,6 +24,7 @@
import com.google.devtools.build.lib.actions.cache.DigestUtils;
import com.google.devtools.build.lib.actions.cache.Metadata;
import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.EventKind;
@@ -285,32 +286,38 @@
if (unconditionalExecution(action)) {
Preconditions.checkState(action.isVolatile());
reportUnconditionalExecution(handler, action);
- return true; // must execute - unconditional execution is requested.
+ actionCache.accountMiss(MissReason.UNCONDITIONAL_EXECUTION);
+ return true;
}
if (entry == null) {
reportNewAction(handler, action);
- return true; // must execute -- no cache entry (e.g. first build)
+ actionCache.accountMiss(MissReason.NOT_CACHED);
+ return true;
}
if (entry.isCorrupted()) {
reportCorruptedCacheEntry(handler, action);
- return true; // cache entry is corrupted - must execute
+ actionCache.accountMiss(MissReason.CORRUPTED_CACHE_ENTRY);
+ return true;
} else if (validateArtifacts(entry, action, actionInputs, metadataHandler, true)) {
reportChanged(handler, action);
- return true; // files have changed
+ actionCache.accountMiss(MissReason.DIFFERENT_FILES);
+ return true;
} else if (!entry.getActionKey().equals(action.getKey())) {
reportCommand(handler, action);
- return true; // must execute -- action key is different
+ actionCache.accountMiss(MissReason.DIFFERENT_ACTION_KEY);
+ return true;
}
Map<String, String> usedClientEnv = computeUsedClientEnv(action, clientEnv);
if (!entry.getUsedClientEnvDigest().equals(DigestUtils.fromEnv(usedClientEnv))) {
reportClientEnv(handler, action, usedClientEnv);
- return true; // different values taken from the environment -- must execute
+ actionCache.accountMiss(MissReason.DIFFERENT_ENVIRONMENT);
+ return true;
}
-
entry.getFileDigest();
- return false; // cache hit
+ actionCache.accountHit();
+ return false;
}
private static Metadata getMetadataOrConstant(MetadataHandler metadataHandler, Artifact artifact)
@@ -460,13 +467,16 @@
if (entry != null) {
if (entry.isCorrupted()) {
reportCorruptedCacheEntry(handler, action);
+ actionCache.accountMiss(MissReason.CORRUPTED_CACHE_ENTRY);
changed = true;
} else if (validateArtifacts(entry, action, action.getInputs(), metadataHandler, false)) {
reportChanged(handler, action);
+ actionCache.accountMiss(MissReason.DIFFERENT_FILES);
changed = true;
}
} else {
reportChangedDeps(handler, action);
+ actionCache.accountMiss(MissReason.DIFFERENT_DEPS);
changed = true;
}
if (changed) {
@@ -482,6 +492,8 @@
metadataHandler.setDigestForVirtualArtifact(middleman, entry.getFileDigest());
if (changed) {
actionCache.put(cacheKey, entry);
+ } else {
+ actionCache.accountHit();
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BUILD b/src/main/java/com/google/devtools/build/lib/actions/BUILD
index db44f36..f481e5f 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/actions/BUILD
@@ -35,6 +35,7 @@
"//src/main/java/com/google/devtools/build/skyframe",
"//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
"//src/main/java/com/google/devtools/common/options",
+ "//src/main/protobuf:action_cache_java_proto",
"//src/main/protobuf:extra_actions_base_java_proto",
"//third_party:auto_value",
"//third_party:guava",
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
index 59eb31f..fd1756a0 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
@@ -15,7 +15,10 @@
package com.google.devtools.build.lib.actions.cache;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.vfs.PathFragment;
@@ -67,6 +70,10 @@
* will continue to return same result regardless of internal data transformations).
*/
final class Entry {
+ /** Unique instance to represent a corrupted cache entry. */
+ public static final ActionCache.Entry CORRUPTED =
+ new ActionCache.Entry(null, ImmutableMap.<String, String>of(), false);
+
private final String actionKey;
@Nullable
// Null iff the corresponding action does not do input discovery.
@@ -141,7 +148,7 @@
* Returns true if this cache entry is corrupted and should be ignored.
*/
public boolean isCorrupted() {
- return actionKey == null;
+ return this == CORRUPTED;
}
/**
@@ -194,4 +201,22 @@
* Dumps action cache content into the given PrintStream.
*/
void dump(PrintStream out);
+
+ /** Accounts one cache hit. */
+ void accountHit();
+
+ /** Accounts one cache miss for the given reason. */
+ void accountMiss(MissReason reason);
+
+ /**
+ * Populates the given builder with statistics.
+ *
+ * <p>The extracted values are not guaranteed to be a consistent snapshot of the metrics tracked
+ * by the action cache. Therefore, even if it is safe to call this function at any point in time,
+ * this should only be called once there are no actions running.
+ */
+ void mergeIntoActionCacheStatistics(ActionCacheStatistics.Builder builder);
+
+ /** Resets the current statistics to zero. */
+ void resetStatistics();
}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
index 70015af..52cca10 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
@@ -16,7 +16,8 @@
import static java.nio.charset.StandardCharsets.ISO_8859_1;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason;
import com.google.devtools.build.lib.clock.Clock;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe;
import com.google.devtools.build.lib.profiler.AutoProfiler;
@@ -35,9 +36,11 @@
import java.nio.BufferUnderflowException;
import java.nio.ByteBuffer;
import java.util.Collection;
+import java.util.EnumMap;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Logger;
/**
@@ -154,8 +157,9 @@
private final PersistentMap<Integer, byte[]> map;
private final PersistentStringIndexer indexer;
- static final ActionCache.Entry CORRUPTED =
- new ActionCache.Entry(null, ImmutableMap.<String, String>of(), false);
+
+ private final AtomicInteger hits = new AtomicInteger();
+ private final Map<MissReason, AtomicInteger> misses = new EnumMap<>(MissReason.class);
public CompactPersistentActionCache(Path cacheRoot, Clock clock) throws IOException {
Path cacheFile = cacheFile(cacheRoot);
@@ -187,6 +191,15 @@
throw new IOException("Failed action cache referential integrity check: " + integrityError);
}
}
+
+ for (MissReason reason : MissReason.values()) {
+ if (reason == MissReason.UNRECOGNIZED) {
+ // The presence of this enum value is a protobuf artifact and confuses our metrics
+ // externalization code below. Just skip it.
+ continue;
+ }
+ misses.put(reason, new AtomicInteger(0));
+ }
}
/**
@@ -254,7 +267,7 @@
return data != null ? CompactPersistentActionCache.decode(indexer, data) : null;
} catch (IOException e) {
// return entry marked as corrupted.
- return CORRUPTED;
+ return ActionCache.Entry.CORRUPTED;
}
}
@@ -420,7 +433,7 @@
if (source.remaining() > 0) {
throw new IOException("serialized entry data has not been fully decoded");
}
- return new Entry(
+ return new ActionCache.Entry(
actionKey,
usedClientEnvDigest,
count == NO_INPUT_DISCOVERY_COUNT ? null : builder.build(),
@@ -429,4 +442,38 @@
throw new IOException("encoded entry data is incomplete", e);
}
}
+
+ @Override
+ public void accountHit() {
+ hits.incrementAndGet();
+ }
+
+ @Override
+ public void accountMiss(MissReason reason) {
+ AtomicInteger counter = misses.get(reason);
+ Preconditions.checkNotNull(counter, "Miss reason %s was not registered in the misses map "
+ + "during cache construction", reason);
+ counter.incrementAndGet();
+ }
+
+ @Override
+ public void mergeIntoActionCacheStatistics(ActionCacheStatistics.Builder builder) {
+ builder.setHits(hits.get());
+
+ int totalMisses = 0;
+ for (Map.Entry<MissReason, AtomicInteger> entry : misses.entrySet()) {
+ int count = entry.getValue().get();
+ builder.addMissDetailsBuilder().setReason(entry.getKey()).setCount(count);
+ totalMisses += count;
+ }
+ builder.setMisses(totalMisses);
+ }
+
+ @Override
+ public void resetStatistics() {
+ hits.set(0);
+ for (Map.Entry<MissReason, AtomicInteger> entry : misses.entrySet()) {
+ entry.getValue().set(0);
+ }
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
index 1285961..af7659b 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -380,6 +380,7 @@
request.getBuildOptions().getSymlinkPrefix(productName), productName);
ActionCache actionCache = getActionCache();
+ actionCache.resetStatistics();
SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
Builder builder = createBuilder(
request, actionCache, skyframeExecutor, modifiedOutputFiles);
@@ -734,6 +735,7 @@
*/
private void saveActionCache(ActionCache actionCache) {
ActionCacheStatistics.Builder builder = ActionCacheStatistics.newBuilder();
+ actionCache.mergeIntoActionCacheStatistics(builder);
AutoProfiler p =
AutoProfiler.profiledAndLogged("Saving action cache", ProfilerTask.INFO, logger);
diff --git a/src/main/protobuf/action_cache.proto b/src/main/protobuf/action_cache.proto
index 429990d..5694ebe 100644
--- a/src/main/protobuf/action_cache.proto
+++ b/src/main/protobuf/action_cache.proto
@@ -31,5 +31,29 @@
// Time it took to save the action cache to disk.
uint64 save_time_in_ms = 2;
- // NEXT TAG: 3
+ // Reasons for not finding an action in the cache.
+ enum MissReason {
+ DIFFERENT_ACTION_KEY = 0;
+ DIFFERENT_DEPS = 1;
+ DIFFERENT_ENVIRONMENT = 2;
+ DIFFERENT_FILES = 3;
+ CORRUPTED_CACHE_ENTRY = 4;
+ NOT_CACHED = 5;
+ UNCONDITIONAL_EXECUTION = 6;
+ }
+
+ // Detailed information for a particular miss reason.
+ message MissDetail {
+ MissReason reason = 1;
+ int32 count = 2;
+ }
+
+ // Cache counters.
+ int32 hits = 3;
+ int32 misses = 4;
+
+ // Breakdown of the cache misses based on the reasons behind them.
+ repeated MissDetail miss_details = 5;
+
+ // NEXT TAG: 6
}
diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index 403db1d..a3542c5 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -289,6 +289,7 @@
"//src/main/java/com/google/devtools/build/skyframe",
"//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
"//src/main/java/com/google/devtools/common/options",
+ "//src/main/protobuf:action_cache_java_proto",
"//third_party:guava",
"//third_party:guava-testlib",
"//third_party:jsr305",
@@ -326,6 +327,7 @@
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
"//src/main/java/com/google/devtools/common/options",
+ "//src/main/protobuf:action_cache_java_proto",
"//third_party:guava",
"//third_party:guava-testlib",
"//third_party:jsr305",
diff --git a/src/test/java/com/google/devtools/build/lib/actions/ActionCacheCheckerTest.java b/src/test/java/com/google/devtools/build/lib/actions/ActionCacheCheckerTest.java
new file mode 100644
index 0000000..4da84f8
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/actions/ActionCacheCheckerTest.java
@@ -0,0 +1,340 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.actions;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.ActionCacheChecker.Token;
+import com.google.devtools.build.lib.actions.cache.ActionCache;
+import com.google.devtools.build.lib.actions.cache.CompactPersistentActionCache;
+import com.google.devtools.build.lib.actions.cache.Md5Digest;
+import com.google.devtools.build.lib.actions.cache.Metadata;
+import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissDetail;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil.FakeArtifactResolverBase;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil.FakeMetadataHandlerBase;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil.MissDetailsBuilder;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil.NullAction;
+import com.google.devtools.build.lib.clock.Clock;
+import com.google.devtools.build.lib.skyframe.FileArtifactValue;
+import com.google.devtools.build.lib.testutil.ManualClock;
+import com.google.devtools.build.lib.testutil.Scratch;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class ActionCacheCheckerTest {
+ private CorruptibleCompactPersistentActionCache cache;
+ private ActionCacheChecker cacheChecker;
+ private Set<Path> filesToDelete;
+
+ @Before
+ public void setupCache() throws Exception {
+ Scratch scratch = new Scratch();
+ Clock clock = new ManualClock();
+ ArtifactResolver artifactResolver = new FakeArtifactResolverBase();
+
+ cache = new CorruptibleCompactPersistentActionCache(scratch.resolve("/cache/test.dat"), clock);
+ cacheChecker = new ActionCacheChecker(cache, artifactResolver, Predicates.alwaysTrue(), null);
+ }
+
+ @Before
+ public void clearFilesToDeleteAfterTest() throws Exception {
+ filesToDelete = new HashSet<>();
+ }
+
+ @After
+ public void deleteFilesCreatedDuringTest() throws Exception {
+ for (Path path : filesToDelete) {
+ path.delete();
+ }
+ }
+
+ /** "Executes" the given action from the point of view of the cache's lifecycle. */
+ private void runAction(Action action) throws Exception {
+ runAction(action, new HashMap<>());
+ }
+
+ /**
+ * "Executes" the given action from the point of view of the cache's lifecycle with a custom
+ * client environment.
+ */
+ private void runAction(Action action, Map<String, String> clientEnv) throws Exception {
+ MetadataHandler metadataHandler = new FakeMetadataHandler();
+
+ for (Artifact artifact : action.getOutputs()) {
+ Path path = artifact.getPath();
+
+ // Record all action outputs as files to be deleted across tests to prevent cross-test
+ // pollution. We need to do this on a path basis because we don't know upfront which file
+ // system they live in so we cannot just recreate the file system. (E.g. all NullActions
+ // share an in-memory file system to hold dummy outputs.)
+ filesToDelete.add(path);
+
+ if (!path.exists()) {
+ FileSystemUtils.writeContentAsLatin1(path, "");
+ }
+ }
+
+ Token token = cacheChecker.getTokenIfNeedToExecute(
+ action, null, clientEnv, null, metadataHandler);
+ if (token != null) {
+ // Real action execution would happen here.
+ cacheChecker.afterExecution(action, token, metadataHandler, clientEnv);
+ }
+ }
+
+ /** Ensures that the cache statistics match exactly the given values. */
+ private void assertStatistics(int hits, Iterable<MissDetail> misses) {
+ ActionCacheStatistics.Builder builder = ActionCacheStatistics.newBuilder();
+ cache.mergeIntoActionCacheStatistics(builder);
+ ActionCacheStatistics stats = builder.build();
+
+ assertThat(stats.getHits()).isEqualTo(hits);
+ assertThat(stats.getMissDetailsList()).containsExactlyElementsIn(misses);
+ }
+
+ private void doTestNotCached(Action action, MissReason missReason) throws Exception {
+ runAction(action);
+
+ assertStatistics(0, new MissDetailsBuilder().set(missReason, 1).build());
+ }
+
+ private void doTestCached(Action action, MissReason missReason) throws Exception {
+ int runs = 5;
+ for (int i = 0; i < runs; i++) {
+ runAction(action);
+ }
+
+ assertStatistics(runs - 1, new MissDetailsBuilder().set(missReason, 1).build());
+ }
+
+ private void doTestCorruptedCacheEntry(Action action) throws Exception {
+ cache.corruptAllEntries();
+ runAction(action);
+
+ assertStatistics(
+ 0,
+ new MissDetailsBuilder().set(MissReason.CORRUPTED_CACHE_ENTRY, 1).build());
+ }
+
+ @Test
+ public void testNoActivity() throws Exception {
+ assertStatistics(0, new MissDetailsBuilder().build());
+ }
+
+ @Test
+ public void testNotCached() throws Exception {
+ doTestNotCached(new NullAction(), MissReason.NOT_CACHED);
+ }
+
+ @Test
+ public void testCached() throws Exception {
+ doTestCached(new NullAction(), MissReason.NOT_CACHED);
+ }
+
+ @Test
+ public void testCorruptedCacheEntry() throws Exception {
+ doTestCorruptedCacheEntry(new NullAction());
+ }
+
+ @Test
+ public void testDifferentActionKey() throws Exception {
+ Action action = new NullAction() {
+ @Override
+ protected String computeKey() {
+ return "key1";
+ }
+ };
+ runAction(action);
+ action = new NullAction() {
+ @Override
+ protected String computeKey() {
+ return "key2";
+ }
+ };
+ runAction(action);
+
+ assertStatistics(
+ 0,
+ new MissDetailsBuilder()
+ .set(MissReason.DIFFERENT_ACTION_KEY, 1)
+ .set(MissReason.NOT_CACHED, 1)
+ .build());
+ }
+
+ @Test
+ public void testDifferentEnvironment() throws Exception {
+ Action action = new NullAction() {
+ @Override
+ public Iterable<String> getClientEnvironmentVariables() {
+ return ImmutableList.of("used-var");
+ }
+ };
+ Map<String, String> clientEnv = new HashMap<>();
+ clientEnv.put("unused-var", "1");
+ runAction(action, clientEnv); // Not cached.
+ clientEnv.remove("unused-var");
+ runAction(action, clientEnv); // Cache hit because we only modified uninteresting variables.
+ clientEnv.put("used-var", "2");
+ runAction(action, clientEnv); // Cache miss because of different environment.
+ runAction(action, clientEnv); // Cache hit because we did not change anything.
+
+ assertStatistics(
+ 2,
+ new MissDetailsBuilder()
+ .set(MissReason.DIFFERENT_ENVIRONMENT, 1)
+ .set(MissReason.NOT_CACHED, 1)
+ .build());
+ }
+
+ @Test
+ public void testDifferentFiles() throws Exception {
+ Action action = new NullAction();
+ runAction(action); // Not cached.
+ FileSystemUtils.writeContentAsLatin1(action.getPrimaryOutput().getPath(), "modified");
+ runAction(action); // Cache miss because output files were modified.
+
+ assertStatistics(
+ 0,
+ new MissDetailsBuilder()
+ .set(MissReason.DIFFERENT_FILES, 1)
+ .set(MissReason.NOT_CACHED, 1)
+ .build());
+ }
+
+ @Test
+ public void testUnconditionalExecution() throws Exception {
+ Action action = new NullAction() {
+ @Override
+ public boolean executeUnconditionally() {
+ return true;
+ }
+
+ @Override
+ public boolean isVolatile() {
+ return true;
+ }
+ };
+
+ int runs = 5;
+ for (int i = 0; i < runs; i++) {
+ runAction(action);
+ }
+
+ assertStatistics(
+ 0, new MissDetailsBuilder().set(MissReason.UNCONDITIONAL_EXECUTION, runs).build());
+ }
+
+ @Test
+ public void testMiddleman_NotCached() throws Exception {
+ doTestNotCached(new NullMiddlemanAction(), MissReason.DIFFERENT_DEPS);
+ }
+
+ @Test
+ public void testMiddleman_Cached() throws Exception {
+ doTestCached(new NullMiddlemanAction(), MissReason.DIFFERENT_DEPS);
+ }
+
+ @Test
+ public void testMiddleman_CorruptedCacheEntry() throws Exception {
+ doTestCorruptedCacheEntry(new NullMiddlemanAction());
+ }
+
+ @Test
+ public void testMiddleman_DifferentFiles() throws Exception {
+ Action action = new NullMiddlemanAction() {
+ @Override
+ public synchronized Iterable<Artifact> getInputs() {
+ FileSystem fileSystem = getPrimaryOutput().getPath().getFileSystem();
+ Path path = fileSystem.getPath("/input");
+ Root root = Root.asSourceRoot(fileSystem.getPath("/"));
+ return ImmutableList.of(new Artifact(path, root));
+ }
+ };
+ runAction(action); // Not cached so recorded as different deps.
+ FileSystemUtils.writeContentAsLatin1(action.getPrimaryInput().getPath(), "modified");
+ runAction(action); // Cache miss because input files were modified.
+ FileSystemUtils.writeContentAsLatin1(action.getPrimaryOutput().getPath(), "modified");
+ runAction(action); // Outputs are not considered for middleman actions, so this is a cache hit.
+ runAction(action); // Outputs are not considered for middleman actions, so this is a cache hit.
+
+ assertStatistics(
+ 2,
+ new MissDetailsBuilder()
+ .set(MissReason.DIFFERENT_DEPS, 1)
+ .set(MissReason.DIFFERENT_FILES, 1)
+ .build());
+ }
+
+ /** A {@link CompactPersistentActionCache} that allows injecting corruption for testing. */
+ private static class CorruptibleCompactPersistentActionCache
+ extends CompactPersistentActionCache {
+ private boolean corrupted = false;
+
+ CorruptibleCompactPersistentActionCache(Path cacheRoot, Clock clock) throws IOException {
+ super(cacheRoot, clock);
+ }
+
+ void corruptAllEntries() {
+ corrupted = true;
+ }
+
+ @Override
+ public Entry get(String key) {
+ if (corrupted) {
+ return ActionCache.Entry.CORRUPTED;
+ } else {
+ return super.get(key);
+ }
+ }
+ }
+
+ /** A null middleman action. */
+ private static class NullMiddlemanAction extends NullAction {
+ @Override
+ public MiddlemanType getActionType() {
+ return MiddlemanType.RUNFILES_MIDDLEMAN;
+ }
+ }
+
+ /** A fake metadata handler that is able to obtain metadata from the file system. */
+ private static class FakeMetadataHandler extends FakeMetadataHandlerBase {
+ @Override
+ public Metadata getMetadata(Artifact artifact) throws IOException {
+ return FileArtifactValue.create(artifact);
+ }
+
+ @Override
+ public void setDigestForVirtualArtifact(Artifact artifact, Md5Digest md5Digest) {
+
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java b/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java
index 1415c81..93fd34f 100644
--- a/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java
+++ b/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java
@@ -14,6 +14,8 @@
package com.google.devtools.build.lib.actions.util;
import com.google.devtools.build.lib.actions.cache.ActionCache;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason;
import java.io.PrintStream;
/**
@@ -46,5 +48,17 @@
@Override
public void dump(PrintStream out) {}
+
+ @Override
+ public void accountHit() {}
+
+ @Override
+ public void accountMiss(MissReason reason) {}
+
+ @Override
+ public void mergeIntoActionCacheStatistics(ActionCacheStatistics.Builder builder) {}
+
+ @Override
+ public void resetStatistics() {}
};
}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
index d1b796b..2fc80f5 100644
--- a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
+++ b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
@@ -14,6 +14,7 @@
package com.google.devtools.build.lib.actions.util;
import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.lib.util.Preconditions.checkArgument;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
@@ -29,6 +30,7 @@
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionGraph;
+import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputHelper;
import com.google.devtools.build.lib.actions.ActionInputPrefetcher;
import com.google.devtools.build.lib.actions.ActionOwner;
@@ -36,15 +38,22 @@
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.ArtifactOwner;
+import com.google.devtools.build.lib.actions.ArtifactResolver;
import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.MutableActionGraph;
import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
+import com.google.devtools.build.lib.actions.PackageRootResolver;
import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.actions.cache.Md5Digest;
+import com.google.devtools.build.lib.actions.cache.Metadata;
import com.google.devtools.build.lib.actions.cache.MetadataHandler;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissDetail;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason;
import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate;
import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate.OutputPathMapper;
import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.events.Reporter;
@@ -54,6 +63,7 @@
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.util.ResourceUsage;
import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
@@ -65,9 +75,11 @@
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.ValueOrExceptionUtils;
import com.google.devtools.build.skyframe.ValueOrUntypedException;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.EnumMap;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
@@ -570,4 +582,120 @@
})
.build(NULL_ACTION_OWNER);
}
+
+ /** Builder for a list of {@link MissDetail}s with defaults set to zero for all possible items. */
+ public static class MissDetailsBuilder {
+ private final Map<MissReason, Integer> details = new EnumMap<>(MissReason.class);
+
+ /** Constructs a new builder with all possible cache miss reasons set to zero counts. */
+ public MissDetailsBuilder() {
+ for (MissReason reason : MissReason.values()) {
+ if (reason == MissReason.UNRECOGNIZED) {
+ // The presence of this enum value is a protobuf artifact and not part of our metrics
+ // collection. Just skip it.
+ continue;
+ }
+ details.put(reason, 0);
+ }
+ }
+
+ /** Sets the count of the given miss reason to the given value. */
+ public MissDetailsBuilder set(MissReason reason, int count) {
+ checkArgument(details.containsKey(reason));
+ details.put(reason, count);
+ return this;
+ }
+
+ /** Constructs the list of {@link MissDetail}s. */
+ public Iterable<MissDetail> build() {
+ List<MissDetail> result = new ArrayList<>(details.size());
+ for (Map.Entry<MissReason, Integer> entry : details.entrySet()) {
+ MissDetail detail = MissDetail.newBuilder()
+ .setReason(entry.getKey())
+ .setCount(entry.getValue())
+ .build();
+ result.add(detail);
+ }
+ return result;
+ }
+ }
+
+ /**
+ * An {@link ArtifactResolver} all of whose operations throw an exception.
+ *
+ * <p>This is to be used as a base class by other test programs that need to implement only a
+ * few of the hooks required by the scenario under test.
+ */
+ public static class FakeArtifactResolverBase implements ArtifactResolver {
+ @Override
+ public Artifact getSourceArtifact(
+ PathFragment execPath, Root root, ArtifactOwner owner) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Artifact getSourceArtifact(PathFragment execPath, Root root) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Artifact resolveSourceArtifact(
+ PathFragment execPath, RepositoryName repositoryName) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Map<PathFragment, Artifact> resolveSourceArtifacts(
+ Iterable<PathFragment> execPaths, PackageRootResolver resolver) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ /**
+ * A {@link MetadataHandler} all of whose operations throw an exception.
+ *
+ * <p>This is to be used as a base class by other test programs that need to implement only a
+ * few of the hooks required by the scenario under test.
+ */
+ public static class FakeMetadataHandlerBase implements MetadataHandler {
+ @Override
+ public Metadata getMetadata(Artifact artifact) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setDigestForVirtualArtifact(Artifact artifact, Md5Digest md5Digest) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void addExpandedTreeOutput(TreeFileArtifact output) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Iterable<TreeFileArtifact> getExpandedOutputs(Artifact artifact) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void markOmitted(ActionInput output) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean artifactOmitted(Artifact artifact) {
+ return false;
+ }
+
+ @Override
+ public void discardOutputMetadata() {
+ throw new UnsupportedOperationException();
+ }
+ }
}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/BUILD b/src/test/java/com/google/devtools/build/lib/skyframe/BUILD
index a25cdbd..259a508 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/BUILD
@@ -82,6 +82,7 @@
"//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
"//src/main/java/com/google/devtools/build/skyframe",
"//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+ "//src/main/protobuf:action_cache_java_proto",
"//src/test/java/com/google/devtools/build/lib:actions_testutil",
"//src/test/java/com/google/devtools/build/lib:analysis_testutil",
"//src/test/java/com/google/devtools/build/lib:foundations_testutil",
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java
index 22033a3..15d0b04 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java
@@ -44,6 +44,8 @@
import com.google.devtools.build.lib.actions.Root;
import com.google.devtools.build.lib.actions.TestExecException;
import com.google.devtools.build.lib.actions.cache.ActionCache;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics;
+import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason;
import com.google.devtools.build.lib.actions.util.DummyExecutor;
import com.google.devtools.build.lib.actions.util.TestAction;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
@@ -458,6 +460,26 @@
public void dump(PrintStream out) {
out.println("In-memory action cache has " + actionCache.size() + " records");
}
+
+ @Override
+ public void accountHit() {
+ // Not needed for these tests.
+ }
+
+ @Override
+ public void accountMiss(MissReason reason) {
+ // Not needed for these tests.
+ }
+
+ @Override
+ public void mergeIntoActionCacheStatistics(ActionCacheStatistics.Builder builder) {
+ // Not needed for these tests.
+ }
+
+ @Override
+ public void resetStatistics() {
+ // Not needed for these tests.
+ }
}
private static class SingletonActionLookupKey extends ActionLookupValue.ActionLookupKey {
