Collect action cache hits, misses, and reasons for the misses. As a bonus, this brings in a bunch of new unit tests for the ActionCacheChecker. RELNOTES: None. PiperOrigin-RevId: 170059577
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java index 1a6c4c8..fc64c75 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java
@@ -24,6 +24,7 @@ import com.google.devtools.build.lib.actions.cache.DigestUtils; import com.google.devtools.build.lib.actions.cache.Metadata; import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.EventKind; @@ -285,32 +286,38 @@ if (unconditionalExecution(action)) { Preconditions.checkState(action.isVolatile()); reportUnconditionalExecution(handler, action); - return true; // must execute - unconditional execution is requested. + actionCache.accountMiss(MissReason.UNCONDITIONAL_EXECUTION); + return true; } if (entry == null) { reportNewAction(handler, action); - return true; // must execute -- no cache entry (e.g. first build) + actionCache.accountMiss(MissReason.NOT_CACHED); + return true; } if (entry.isCorrupted()) { reportCorruptedCacheEntry(handler, action); - return true; // cache entry is corrupted - must execute + actionCache.accountMiss(MissReason.CORRUPTED_CACHE_ENTRY); + return true; } else if (validateArtifacts(entry, action, actionInputs, metadataHandler, true)) { reportChanged(handler, action); - return true; // files have changed + actionCache.accountMiss(MissReason.DIFFERENT_FILES); + return true; } else if (!entry.getActionKey().equals(action.getKey())) { reportCommand(handler, action); - return true; // must execute -- action key is different + actionCache.accountMiss(MissReason.DIFFERENT_ACTION_KEY); + return true; } Map<String, String> usedClientEnv = computeUsedClientEnv(action, clientEnv); if (!entry.getUsedClientEnvDigest().equals(DigestUtils.fromEnv(usedClientEnv))) { reportClientEnv(handler, action, usedClientEnv); - return true; // different values taken from the environment -- must execute + actionCache.accountMiss(MissReason.DIFFERENT_ENVIRONMENT); + return true; } - entry.getFileDigest(); - return false; // cache hit + actionCache.accountHit(); + return false; } private static Metadata getMetadataOrConstant(MetadataHandler metadataHandler, Artifact artifact) @@ -460,13 +467,16 @@ if (entry != null) { if (entry.isCorrupted()) { reportCorruptedCacheEntry(handler, action); + actionCache.accountMiss(MissReason.CORRUPTED_CACHE_ENTRY); changed = true; } else if (validateArtifacts(entry, action, action.getInputs(), metadataHandler, false)) { reportChanged(handler, action); + actionCache.accountMiss(MissReason.DIFFERENT_FILES); changed = true; } } else { reportChangedDeps(handler, action); + actionCache.accountMiss(MissReason.DIFFERENT_DEPS); changed = true; } if (changed) { @@ -482,6 +492,8 @@ metadataHandler.setDigestForVirtualArtifact(middleman, entry.getFileDigest()); if (changed) { actionCache.put(cacheKey, entry); + } else { + actionCache.accountHit(); } }
diff --git a/src/main/java/com/google/devtools/build/lib/actions/BUILD b/src/main/java/com/google/devtools/build/lib/actions/BUILD index db44f36..f481e5f 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/BUILD +++ b/src/main/java/com/google/devtools/build/lib/actions/BUILD
@@ -35,6 +35,7 @@ "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//src/main/java/com/google/devtools/common/options", + "//src/main/protobuf:action_cache_java_proto", "//src/main/protobuf:extra_actions_base_java_proto", "//third_party:auto_value", "//third_party:guava",
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java index 59eb31f..fd1756a0 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
@@ -15,7 +15,10 @@ package com.google.devtools.build.lib.actions.cache; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.vfs.PathFragment; @@ -67,6 +70,10 @@ * will continue to return same result regardless of internal data transformations). */ final class Entry { + /** Unique instance to represent a corrupted cache entry. */ + public static final ActionCache.Entry CORRUPTED = + new ActionCache.Entry(null, ImmutableMap.<String, String>of(), false); + private final String actionKey; @Nullable // Null iff the corresponding action does not do input discovery. @@ -141,7 +148,7 @@ * Returns true if this cache entry is corrupted and should be ignored. */ public boolean isCorrupted() { - return actionKey == null; + return this == CORRUPTED; } /** @@ -194,4 +201,22 @@ * Dumps action cache content into the given PrintStream. */ void dump(PrintStream out); + + /** Accounts one cache hit. */ + void accountHit(); + + /** Accounts one cache miss for the given reason. */ + void accountMiss(MissReason reason); + + /** + * Populates the given builder with statistics. + * + * <p>The extracted values are not guaranteed to be a consistent snapshot of the metrics tracked + * by the action cache. Therefore, even if it is safe to call this function at any point in time, + * this should only be called once there are no actions running. + */ + void mergeIntoActionCacheStatistics(ActionCacheStatistics.Builder builder); + + /** Resets the current statistics to zero. */ + void resetStatistics(); }
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java index 70015af..52cca10 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
@@ -16,7 +16,8 @@ import static java.nio.charset.StandardCharsets.ISO_8859_1; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason; import com.google.devtools.build.lib.clock.Clock; import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe; import com.google.devtools.build.lib.profiler.AutoProfiler; @@ -35,9 +36,11 @@ import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.Collection; +import java.util.EnumMap; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; /** @@ -154,8 +157,9 @@ private final PersistentMap<Integer, byte[]> map; private final PersistentStringIndexer indexer; - static final ActionCache.Entry CORRUPTED = - new ActionCache.Entry(null, ImmutableMap.<String, String>of(), false); + + private final AtomicInteger hits = new AtomicInteger(); + private final Map<MissReason, AtomicInteger> misses = new EnumMap<>(MissReason.class); public CompactPersistentActionCache(Path cacheRoot, Clock clock) throws IOException { Path cacheFile = cacheFile(cacheRoot); @@ -187,6 +191,15 @@ throw new IOException("Failed action cache referential integrity check: " + integrityError); } } + + for (MissReason reason : MissReason.values()) { + if (reason == MissReason.UNRECOGNIZED) { + // The presence of this enum value is a protobuf artifact and confuses our metrics + // externalization code below. Just skip it. + continue; + } + misses.put(reason, new AtomicInteger(0)); + } } /** @@ -254,7 +267,7 @@ return data != null ? CompactPersistentActionCache.decode(indexer, data) : null; } catch (IOException e) { // return entry marked as corrupted. - return CORRUPTED; + return ActionCache.Entry.CORRUPTED; } } @@ -420,7 +433,7 @@ if (source.remaining() > 0) { throw new IOException("serialized entry data has not been fully decoded"); } - return new Entry( + return new ActionCache.Entry( actionKey, usedClientEnvDigest, count == NO_INPUT_DISCOVERY_COUNT ? null : builder.build(), @@ -429,4 +442,38 @@ throw new IOException("encoded entry data is incomplete", e); } } + + @Override + public void accountHit() { + hits.incrementAndGet(); + } + + @Override + public void accountMiss(MissReason reason) { + AtomicInteger counter = misses.get(reason); + Preconditions.checkNotNull(counter, "Miss reason %s was not registered in the misses map " + + "during cache construction", reason); + counter.incrementAndGet(); + } + + @Override + public void mergeIntoActionCacheStatistics(ActionCacheStatistics.Builder builder) { + builder.setHits(hits.get()); + + int totalMisses = 0; + for (Map.Entry<MissReason, AtomicInteger> entry : misses.entrySet()) { + int count = entry.getValue().get(); + builder.addMissDetailsBuilder().setReason(entry.getKey()).setCount(count); + totalMisses += count; + } + builder.setMisses(totalMisses); + } + + @Override + public void resetStatistics() { + hits.set(0); + for (Map.Entry<MissReason, AtomicInteger> entry : misses.entrySet()) { + entry.getValue().set(0); + } + } }
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index 1285961..af7659b 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -380,6 +380,7 @@ request.getBuildOptions().getSymlinkPrefix(productName), productName); ActionCache actionCache = getActionCache(); + actionCache.resetStatistics(); SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor(); Builder builder = createBuilder( request, actionCache, skyframeExecutor, modifiedOutputFiles); @@ -734,6 +735,7 @@ */ private void saveActionCache(ActionCache actionCache) { ActionCacheStatistics.Builder builder = ActionCacheStatistics.newBuilder(); + actionCache.mergeIntoActionCacheStatistics(builder); AutoProfiler p = AutoProfiler.profiledAndLogged("Saving action cache", ProfilerTask.INFO, logger);
diff --git a/src/main/protobuf/action_cache.proto b/src/main/protobuf/action_cache.proto index 429990d..5694ebe 100644 --- a/src/main/protobuf/action_cache.proto +++ b/src/main/protobuf/action_cache.proto
@@ -31,5 +31,29 @@ // Time it took to save the action cache to disk. uint64 save_time_in_ms = 2; - // NEXT TAG: 3 + // Reasons for not finding an action in the cache. + enum MissReason { + DIFFERENT_ACTION_KEY = 0; + DIFFERENT_DEPS = 1; + DIFFERENT_ENVIRONMENT = 2; + DIFFERENT_FILES = 3; + CORRUPTED_CACHE_ENTRY = 4; + NOT_CACHED = 5; + UNCONDITIONAL_EXECUTION = 6; + } + + // Detailed information for a particular miss reason. + message MissDetail { + MissReason reason = 1; + int32 count = 2; + } + + // Cache counters. + int32 hits = 3; + int32 misses = 4; + + // Breakdown of the cache misses based on the reasons behind them. + repeated MissDetail miss_details = 5; + + // NEXT TAG: 6 }
diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD index 403db1d..a3542c5 100644 --- a/src/test/java/com/google/devtools/build/lib/BUILD +++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -289,6 +289,7 @@ "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//src/main/java/com/google/devtools/common/options", + "//src/main/protobuf:action_cache_java_proto", "//third_party:guava", "//third_party:guava-testlib", "//third_party:jsr305", @@ -326,6 +327,7 @@ "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs", "//src/main/java/com/google/devtools/common/options", + "//src/main/protobuf:action_cache_java_proto", "//third_party:guava", "//third_party:guava-testlib", "//third_party:jsr305",
diff --git a/src/test/java/com/google/devtools/build/lib/actions/ActionCacheCheckerTest.java b/src/test/java/com/google/devtools/build/lib/actions/ActionCacheCheckerTest.java new file mode 100644 index 0000000..4da84f8 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/ActionCacheCheckerTest.java
@@ -0,0 +1,340 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionCacheChecker.Token; +import com.google.devtools.build.lib.actions.cache.ActionCache; +import com.google.devtools.build.lib.actions.cache.CompactPersistentActionCache; +import com.google.devtools.build.lib.actions.cache.Md5Digest; +import com.google.devtools.build.lib.actions.cache.Metadata; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissDetail; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil.FakeArtifactResolverBase; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil.FakeMetadataHandlerBase; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil.MissDetailsBuilder; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil.NullAction; +import com.google.devtools.build.lib.clock.Clock; +import com.google.devtools.build.lib.skyframe.FileArtifactValue; +import com.google.devtools.build.lib.testutil.ManualClock; +import com.google.devtools.build.lib.testutil.Scratch; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ActionCacheCheckerTest { + private CorruptibleCompactPersistentActionCache cache; + private ActionCacheChecker cacheChecker; + private Set<Path> filesToDelete; + + @Before + public void setupCache() throws Exception { + Scratch scratch = new Scratch(); + Clock clock = new ManualClock(); + ArtifactResolver artifactResolver = new FakeArtifactResolverBase(); + + cache = new CorruptibleCompactPersistentActionCache(scratch.resolve("/cache/test.dat"), clock); + cacheChecker = new ActionCacheChecker(cache, artifactResolver, Predicates.alwaysTrue(), null); + } + + @Before + public void clearFilesToDeleteAfterTest() throws Exception { + filesToDelete = new HashSet<>(); + } + + @After + public void deleteFilesCreatedDuringTest() throws Exception { + for (Path path : filesToDelete) { + path.delete(); + } + } + + /** "Executes" the given action from the point of view of the cache's lifecycle. */ + private void runAction(Action action) throws Exception { + runAction(action, new HashMap<>()); + } + + /** + * "Executes" the given action from the point of view of the cache's lifecycle with a custom + * client environment. + */ + private void runAction(Action action, Map<String, String> clientEnv) throws Exception { + MetadataHandler metadataHandler = new FakeMetadataHandler(); + + for (Artifact artifact : action.getOutputs()) { + Path path = artifact.getPath(); + + // Record all action outputs as files to be deleted across tests to prevent cross-test + // pollution. We need to do this on a path basis because we don't know upfront which file + // system they live in so we cannot just recreate the file system. (E.g. all NullActions + // share an in-memory file system to hold dummy outputs.) + filesToDelete.add(path); + + if (!path.exists()) { + FileSystemUtils.writeContentAsLatin1(path, ""); + } + } + + Token token = cacheChecker.getTokenIfNeedToExecute( + action, null, clientEnv, null, metadataHandler); + if (token != null) { + // Real action execution would happen here. + cacheChecker.afterExecution(action, token, metadataHandler, clientEnv); + } + } + + /** Ensures that the cache statistics match exactly the given values. */ + private void assertStatistics(int hits, Iterable<MissDetail> misses) { + ActionCacheStatistics.Builder builder = ActionCacheStatistics.newBuilder(); + cache.mergeIntoActionCacheStatistics(builder); + ActionCacheStatistics stats = builder.build(); + + assertThat(stats.getHits()).isEqualTo(hits); + assertThat(stats.getMissDetailsList()).containsExactlyElementsIn(misses); + } + + private void doTestNotCached(Action action, MissReason missReason) throws Exception { + runAction(action); + + assertStatistics(0, new MissDetailsBuilder().set(missReason, 1).build()); + } + + private void doTestCached(Action action, MissReason missReason) throws Exception { + int runs = 5; + for (int i = 0; i < runs; i++) { + runAction(action); + } + + assertStatistics(runs - 1, new MissDetailsBuilder().set(missReason, 1).build()); + } + + private void doTestCorruptedCacheEntry(Action action) throws Exception { + cache.corruptAllEntries(); + runAction(action); + + assertStatistics( + 0, + new MissDetailsBuilder().set(MissReason.CORRUPTED_CACHE_ENTRY, 1).build()); + } + + @Test + public void testNoActivity() throws Exception { + assertStatistics(0, new MissDetailsBuilder().build()); + } + + @Test + public void testNotCached() throws Exception { + doTestNotCached(new NullAction(), MissReason.NOT_CACHED); + } + + @Test + public void testCached() throws Exception { + doTestCached(new NullAction(), MissReason.NOT_CACHED); + } + + @Test + public void testCorruptedCacheEntry() throws Exception { + doTestCorruptedCacheEntry(new NullAction()); + } + + @Test + public void testDifferentActionKey() throws Exception { + Action action = new NullAction() { + @Override + protected String computeKey() { + return "key1"; + } + }; + runAction(action); + action = new NullAction() { + @Override + protected String computeKey() { + return "key2"; + } + }; + runAction(action); + + assertStatistics( + 0, + new MissDetailsBuilder() + .set(MissReason.DIFFERENT_ACTION_KEY, 1) + .set(MissReason.NOT_CACHED, 1) + .build()); + } + + @Test + public void testDifferentEnvironment() throws Exception { + Action action = new NullAction() { + @Override + public Iterable<String> getClientEnvironmentVariables() { + return ImmutableList.of("used-var"); + } + }; + Map<String, String> clientEnv = new HashMap<>(); + clientEnv.put("unused-var", "1"); + runAction(action, clientEnv); // Not cached. + clientEnv.remove("unused-var"); + runAction(action, clientEnv); // Cache hit because we only modified uninteresting variables. + clientEnv.put("used-var", "2"); + runAction(action, clientEnv); // Cache miss because of different environment. + runAction(action, clientEnv); // Cache hit because we did not change anything. + + assertStatistics( + 2, + new MissDetailsBuilder() + .set(MissReason.DIFFERENT_ENVIRONMENT, 1) + .set(MissReason.NOT_CACHED, 1) + .build()); + } + + @Test + public void testDifferentFiles() throws Exception { + Action action = new NullAction(); + runAction(action); // Not cached. + FileSystemUtils.writeContentAsLatin1(action.getPrimaryOutput().getPath(), "modified"); + runAction(action); // Cache miss because output files were modified. + + assertStatistics( + 0, + new MissDetailsBuilder() + .set(MissReason.DIFFERENT_FILES, 1) + .set(MissReason.NOT_CACHED, 1) + .build()); + } + + @Test + public void testUnconditionalExecution() throws Exception { + Action action = new NullAction() { + @Override + public boolean executeUnconditionally() { + return true; + } + + @Override + public boolean isVolatile() { + return true; + } + }; + + int runs = 5; + for (int i = 0; i < runs; i++) { + runAction(action); + } + + assertStatistics( + 0, new MissDetailsBuilder().set(MissReason.UNCONDITIONAL_EXECUTION, runs).build()); + } + + @Test + public void testMiddleman_NotCached() throws Exception { + doTestNotCached(new NullMiddlemanAction(), MissReason.DIFFERENT_DEPS); + } + + @Test + public void testMiddleman_Cached() throws Exception { + doTestCached(new NullMiddlemanAction(), MissReason.DIFFERENT_DEPS); + } + + @Test + public void testMiddleman_CorruptedCacheEntry() throws Exception { + doTestCorruptedCacheEntry(new NullMiddlemanAction()); + } + + @Test + public void testMiddleman_DifferentFiles() throws Exception { + Action action = new NullMiddlemanAction() { + @Override + public synchronized Iterable<Artifact> getInputs() { + FileSystem fileSystem = getPrimaryOutput().getPath().getFileSystem(); + Path path = fileSystem.getPath("/input"); + Root root = Root.asSourceRoot(fileSystem.getPath("/")); + return ImmutableList.of(new Artifact(path, root)); + } + }; + runAction(action); // Not cached so recorded as different deps. + FileSystemUtils.writeContentAsLatin1(action.getPrimaryInput().getPath(), "modified"); + runAction(action); // Cache miss because input files were modified. + FileSystemUtils.writeContentAsLatin1(action.getPrimaryOutput().getPath(), "modified"); + runAction(action); // Outputs are not considered for middleman actions, so this is a cache hit. + runAction(action); // Outputs are not considered for middleman actions, so this is a cache hit. + + assertStatistics( + 2, + new MissDetailsBuilder() + .set(MissReason.DIFFERENT_DEPS, 1) + .set(MissReason.DIFFERENT_FILES, 1) + .build()); + } + + /** A {@link CompactPersistentActionCache} that allows injecting corruption for testing. */ + private static class CorruptibleCompactPersistentActionCache + extends CompactPersistentActionCache { + private boolean corrupted = false; + + CorruptibleCompactPersistentActionCache(Path cacheRoot, Clock clock) throws IOException { + super(cacheRoot, clock); + } + + void corruptAllEntries() { + corrupted = true; + } + + @Override + public Entry get(String key) { + if (corrupted) { + return ActionCache.Entry.CORRUPTED; + } else { + return super.get(key); + } + } + } + + /** A null middleman action. */ + private static class NullMiddlemanAction extends NullAction { + @Override + public MiddlemanType getActionType() { + return MiddlemanType.RUNFILES_MIDDLEMAN; + } + } + + /** A fake metadata handler that is able to obtain metadata from the file system. */ + private static class FakeMetadataHandler extends FakeMetadataHandlerBase { + @Override + public Metadata getMetadata(Artifact artifact) throws IOException { + return FileArtifactValue.create(artifact); + } + + @Override + public void setDigestForVirtualArtifact(Artifact artifact, Md5Digest md5Digest) { + + } + } +}
diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java b/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java index 1415c81..93fd34f 100644 --- a/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java +++ b/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java
@@ -14,6 +14,8 @@ package com.google.devtools.build.lib.actions.util; import com.google.devtools.build.lib.actions.cache.ActionCache; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason; import java.io.PrintStream; /** @@ -46,5 +48,17 @@ @Override public void dump(PrintStream out) {} + + @Override + public void accountHit() {} + + @Override + public void accountMiss(MissReason reason) {} + + @Override + public void mergeIntoActionCacheStatistics(ActionCacheStatistics.Builder builder) {} + + @Override + public void resetStatistics() {} }; }
diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java index d1b796b..2fc80f5 100644 --- a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java +++ b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java
@@ -14,6 +14,7 @@ package com.google.devtools.build.lib.actions.util; import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.util.Preconditions.checkArgument; import com.google.common.base.Joiner; import com.google.common.base.Predicate; @@ -29,6 +30,7 @@ import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputHelper; import com.google.devtools.build.lib.actions.ActionInputPrefetcher; import com.google.devtools.build.lib.actions.ActionOwner; @@ -36,15 +38,22 @@ import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.ArtifactResolver; import com.google.devtools.build.lib.actions.Executor; import com.google.devtools.build.lib.actions.MutableActionGraph; import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; +import com.google.devtools.build.lib.actions.PackageRootResolver; import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.actions.cache.Md5Digest; +import com.google.devtools.build.lib.actions.cache.Metadata; import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissDetail; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason; import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate; import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate.OutputPathMapper; import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.events.Reporter; @@ -54,6 +63,7 @@ import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.util.ResourceUsage; import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.FileStatus; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; @@ -65,9 +75,11 @@ import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.build.skyframe.ValueOrExceptionUtils; import com.google.devtools.build.skyframe.ValueOrUntypedException; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.EnumMap; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.LinkedList; @@ -570,4 +582,120 @@ }) .build(NULL_ACTION_OWNER); } + + /** Builder for a list of {@link MissDetail}s with defaults set to zero for all possible items. */ + public static class MissDetailsBuilder { + private final Map<MissReason, Integer> details = new EnumMap<>(MissReason.class); + + /** Constructs a new builder with all possible cache miss reasons set to zero counts. */ + public MissDetailsBuilder() { + for (MissReason reason : MissReason.values()) { + if (reason == MissReason.UNRECOGNIZED) { + // The presence of this enum value is a protobuf artifact and not part of our metrics + // collection. Just skip it. + continue; + } + details.put(reason, 0); + } + } + + /** Sets the count of the given miss reason to the given value. */ + public MissDetailsBuilder set(MissReason reason, int count) { + checkArgument(details.containsKey(reason)); + details.put(reason, count); + return this; + } + + /** Constructs the list of {@link MissDetail}s. */ + public Iterable<MissDetail> build() { + List<MissDetail> result = new ArrayList<>(details.size()); + for (Map.Entry<MissReason, Integer> entry : details.entrySet()) { + MissDetail detail = MissDetail.newBuilder() + .setReason(entry.getKey()) + .setCount(entry.getValue()) + .build(); + result.add(detail); + } + return result; + } + } + + /** + * An {@link ArtifactResolver} all of whose operations throw an exception. + * + * <p>This is to be used as a base class by other test programs that need to implement only a + * few of the hooks required by the scenario under test. + */ + public static class FakeArtifactResolverBase implements ArtifactResolver { + @Override + public Artifact getSourceArtifact( + PathFragment execPath, Root root, ArtifactOwner owner) { + throw new UnsupportedOperationException(); + } + + @Override + public Artifact getSourceArtifact(PathFragment execPath, Root root) { + throw new UnsupportedOperationException(); + } + + @Override + public Artifact resolveSourceArtifact( + PathFragment execPath, RepositoryName repositoryName) { + throw new UnsupportedOperationException(); + } + + @Override + public Map<PathFragment, Artifact> resolveSourceArtifacts( + Iterable<PathFragment> execPaths, PackageRootResolver resolver) { + throw new UnsupportedOperationException(); + } + } + + /** + * A {@link MetadataHandler} all of whose operations throw an exception. + * + * <p>This is to be used as a base class by other test programs that need to implement only a + * few of the hooks required by the scenario under test. + */ + public static class FakeMetadataHandlerBase implements MetadataHandler { + @Override + public Metadata getMetadata(Artifact artifact) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void setDigestForVirtualArtifact(Artifact artifact, Md5Digest md5Digest) { + throw new UnsupportedOperationException(); + } + + @Override + public void addExpandedTreeOutput(TreeFileArtifact output) { + throw new UnsupportedOperationException(); + } + + @Override + public Iterable<TreeFileArtifact> getExpandedOutputs(Artifact artifact) { + throw new UnsupportedOperationException(); + } + + @Override + public void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest) { + throw new UnsupportedOperationException(); + } + + @Override + public void markOmitted(ActionInput output) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean artifactOmitted(Artifact artifact) { + return false; + } + + @Override + public void discardOutputMetadata() { + throw new UnsupportedOperationException(); + } + } }
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/BUILD b/src/test/java/com/google/devtools/build/lib/skyframe/BUILD index a25cdbd..259a508 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/test/java/com/google/devtools/build/lib/skyframe/BUILD
@@ -82,6 +82,7 @@ "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs", "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", + "//src/main/protobuf:action_cache_java_proto", "//src/test/java/com/google/devtools/build/lib:actions_testutil", "//src/test/java/com/google/devtools/build/lib:analysis_testutil", "//src/test/java/com/google/devtools/build/lib:foundations_testutil",
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java index 22033a3..15d0b04 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java
@@ -44,6 +44,8 @@ import com.google.devtools.build.lib.actions.Root; import com.google.devtools.build.lib.actions.TestExecException; import com.google.devtools.build.lib.actions.cache.ActionCache; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics; +import com.google.devtools.build.lib.actions.cache.Protos.ActionCacheStatistics.MissReason; import com.google.devtools.build.lib.actions.util.DummyExecutor; import com.google.devtools.build.lib.actions.util.TestAction; import com.google.devtools.build.lib.analysis.BlazeDirectories; @@ -458,6 +460,26 @@ public void dump(PrintStream out) { out.println("In-memory action cache has " + actionCache.size() + " records"); } + + @Override + public void accountHit() { + // Not needed for these tests. + } + + @Override + public void accountMiss(MissReason reason) { + // Not needed for these tests. + } + + @Override + public void mergeIntoActionCacheStatistics(ActionCacheStatistics.Builder builder) { + // Not needed for these tests. + } + + @Override + public void resetStatistics() { + // Not needed for these tests. + } } private static class SingletonActionLookupKey extends ActionLookupValue.ActionLookupKey {