Open source some skyframe/bazel tests.
--
MOS_MIGRATED_REVID=106308990
diff --git a/src/test/java/BUILD b/src/test/java/BUILD
index 7ab369c..7fbb173 100644
--- a/src/test/java/BUILD
+++ b/src/test/java/BUILD
@@ -34,22 +34,65 @@
],
)
-java_test(
- name = "skyframe_test",
+java_library(
+ name = "skyframe_testutil",
srcs = glob([
- "com/google/devtools/build/skyframe/*.java",
+ "com/google/devtools/build/lib/skyframe/util/*.java",
]),
- args = ["com.google.devtools.build.skyframe.AllTests"],
tags = ["skyframe"],
deps = [
+ ":actions_testutil",
+ ":analysis_testutil",
+ ":foundations_testutil",
":testutil",
+ "//src/main/java:analysis-exec-rules-skyframe",
"//src/main/java:bazel-core",
+ "//src/main/java:clock",
+ "//src/main/java:cmdline",
"//src/main/java:collect",
"//src/main/java:concurrent",
"//src/main/java:events",
+ "//src/main/java:io",
+ "//src/main/java:packages",
"//src/main/java:skyframe-base",
"//src/main/java:util",
"//src/main/java:vfs",
+ "//src/main/java/com/google/devtools/build/lib/actions",
+ "//third_party:guava",
+ "//third_party:guava-testlib",
+ "//third_party:jsr305",
+ "//third_party:junit4",
+ "//third_party:truth",
+ ],
+)
+
+java_test(
+ name = "skyframe_test",
+ srcs = glob([
+ "com/google/devtools/build/lib/skyframe/*.java",
+ ]),
+ args = ["com.google.devtools.build.lib.AllTests"],
+ tags = ["skyframe"],
+ deps = [
+ ":actions_testutil",
+ ":analysis_testutil",
+ ":foundations_testutil",
+ ":skyframe_testutil",
+ ":testutil",
+ "//src/main/java:analysis-exec-rules-skyframe",
+ "//src/main/java:bazel-core",
+ "//src/main/java:buildtool-runtime",
+ "//src/main/java:clock",
+ "//src/main/java:cmdline",
+ "//src/main/java:collect",
+ "//src/main/java:concurrent",
+ "//src/main/java:events",
+ "//src/main/java:io",
+ "//src/main/java:packages",
+ "//src/main/java:skyframe-base",
+ "//src/main/java:util",
+ "//src/main/java:vfs",
+ "//src/main/java/com/google/devtools/build/lib/actions",
"//third_party:guava",
"//third_party:guava-testlib",
"//third_party:jsr305",
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ActionDataTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/ActionDataTest.java
new file mode 100644
index 0000000..be903de
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/ActionDataTest.java
@@ -0,0 +1,173 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.Actions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
+import com.google.devtools.build.lib.actions.util.DummyExecutor;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+
+/**
+ * Tests that the data passed from the application to the Builder is passed
+ * down to each Action executed.
+ */
+public class ActionDataTest extends TimestampBuilderTestCase {
+
+ public void testArgumentToBuildArtifactsIsPassedDownToAction() throws Exception {
+
+ class MyAction extends AbstractAction {
+
+ Object executor = null;
+
+ public MyAction(Collection<Artifact> outputs) {
+ super(ActionsTestUtil.NULL_ACTION_OWNER, ImmutableList.<Artifact>of(), outputs);
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException {
+ this.executor = actionExecutionContext.getExecutor();
+ try {
+ FileSystemUtils.createEmptyFile(getPrimaryOutput().getPath());
+ } catch (IOException e) {
+ throw new ActionExecutionException("failed: ", e, this, false);
+ }
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return ResourceSet.ZERO;
+ }
+
+ @Override
+ protected String computeKey() {
+ return "MyAction";
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "";
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "MyAction";
+ }
+ }
+
+ Artifact output = createDerivedArtifact("foo");
+ Set<Artifact> outputs = Sets.newHashSet(output);
+
+ MyAction action = new MyAction(outputs);
+ registerAction(action);
+
+ Executor executor = new DummyExecutor(scratch.dir("/"));
+ amnesiacBuilder()
+ .buildArtifacts(
+ reporter, outputs, null, null, null, null, executor, null, /*explain=*/ false, null);
+ assertSame(executor, action.executor);
+
+ executor = new DummyExecutor(scratch.dir("/"));
+ amnesiacBuilder()
+ .buildArtifacts(
+ reporter, outputs, null, null, null, null, executor, null, /*explain=*/ false, null);
+ assertSame(executor, action.executor);
+ }
+
+ private static class InputDiscoveringAction extends AbstractAction {
+ private final Collection<Artifact> discoveredInputs;
+
+ public InputDiscoveringAction(Artifact output, Collection<Artifact> discoveredInputs) {
+ super(
+ ActionsTestUtil.NULL_ACTION_OWNER,
+ ImmutableList.<Artifact>of(),
+ ImmutableList.of(output));
+ this.discoveredInputs = discoveredInputs;
+ }
+
+ @Override
+ public boolean discoversInputs() {
+ return true;
+ }
+
+ @Override
+ public boolean inputsKnown() {
+ return true;
+ }
+
+ @Override
+ public Iterable<Artifact> getMandatoryInputs() {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public Iterable<Artifact> getInputs() {
+ return discoveredInputs;
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext) {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return "";
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "InputDiscovering";
+ }
+
+ @Override
+ protected String computeKey() {
+ return "";
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return ResourceSet.ZERO;
+ }
+ }
+
+ public void testActionSharabilityAndDiscoveredInputs() throws Exception {
+ Artifact output =
+ new Artifact(
+ scratch.file("/out/output"), Root.asDerivedRoot(scratch.dir("/"), scratch.dir("/out")));
+ Artifact discovered =
+ new Artifact(
+ scratch.file("/bin/discovered"),
+ Root.asDerivedRoot(scratch.dir("/"), scratch.dir("/bin")));
+
+ Action a = new InputDiscoveringAction(output, ImmutableList.of(discovered));
+ Action b = new InputDiscoveringAction(output, ImmutableList.<Artifact>of());
+
+ assertTrue(Actions.canBeShared(a, b));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdogTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdogTest.java
new file mode 100644
index 0000000..8c15b6c
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/ActionExecutionInactivityWatchdogTest.java
@@ -0,0 +1,167 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.devtools.build.lib.skyframe.ActionExecutionInactivityWatchdog.InactivityMonitor;
+import com.google.devtools.build.lib.skyframe.ActionExecutionInactivityWatchdog.InactivityReporter;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+
+import junit.framework.TestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** Tests for ActionExecutionInactivityWatchdog. */
+public final class ActionExecutionInactivityWatchdogTest extends TestCase {
+
+ private void assertInactivityWatchdogReports(final boolean shouldReport) throws Exception {
+ // The monitor implementation below is a state machine. This variable indicates which state
+ // it is in.
+ final int[] monitorState = new int[] {0};
+
+ // Object that the test thread will wait on.
+ final Object monitorFinishedIndicator = new Object();
+
+ // Reported number of action completions in each call to waitForNextCompletion.
+ final int[] actionCompletions = new int[] {1, 0, 3, 0, 0, 0, 0, 2};
+
+ // Simulated delay of action completions in each call to waitForNextCompletion.
+ final int[] waits = new int[] {5, 10, 3, 10, 30, 60, 60, 1};
+
+ // Log of all Sleep.sleep and InactivityMonitor.waitForNextCompletion calls.
+ final List<String> sleepsAndWaits = new ArrayList<>();
+
+ // Mock monitor for this test.
+ InactivityMonitor monitor =
+ new InactivityMonitor() {
+ @Override
+ public int waitForNextCompletion(int timeoutMilliseconds) throws InterruptedException {
+ // Simulate the following sequence of events (see actionCompletions):
+ // 1. return in 5s (within timeout), 1 action completed; caller will sleep
+ // 2. return in 10s (after timeout), 0 action completed; caller will wait
+ // 3. return in 3s (within timeout), 3 actions completed (this is possible, since the
+ // waiting (thread doesn't necessarily wake up immediately); caller will sleep
+ // 4. return in 10s (after timeout), 0 action completed; caller will wait 30s
+ // 5. return in 30s (after timeout), 0 action completed still; caller will wait 60s
+ // 6. return in 60s (after timeout), 0 action completed still; caller will wait 60s
+ // 7. return in 60s (after timeout), 0 action completed still; caller will wait 60s
+ // 8. return in 1s (within timeout), 2 actions completed; caller will sleep, but we
+ // won't record that, because monitorState reached its maximum
+ synchronized (monitorFinishedIndicator) {
+ if (monitorState[0] >= actionCompletions.length) {
+ // Notify the test thread that the test is over.
+ monitorFinishedIndicator.notify();
+ return 1;
+ } else {
+ int index = monitorState[0];
+ sleepsAndWaits.add("wait:" + waits[index]);
+ ++monitorState[0];
+ return actionCompletions[index];
+ }
+ }
+ }
+
+ @Override
+ public boolean hasStarted() {
+ return true;
+ }
+
+ @Override
+ public int getPending() {
+ int index = monitorState[0];
+ if (index >= actionCompletions.length) {
+ return 0;
+ }
+ int result = actionCompletions[index];
+ while (result == 0) {
+ ++index;
+ result = actionCompletions[index];
+ }
+ return result;
+ }
+ };
+
+ final boolean[] didReportInactivity = new boolean[] {false};
+ InactivityReporter reporter =
+ new InactivityReporter() {
+ @Override
+ public void maybeReportInactivity() {
+ if (shouldReport) {
+ didReportInactivity[0] = true;
+ }
+ }
+ };
+
+ // Mock sleep object; just logs how much the caller's thread would've slept.
+ ActionExecutionInactivityWatchdog.Sleep sleep =
+ new ActionExecutionInactivityWatchdog.Sleep() {
+ @Override
+ public void sleep(int durationMilliseconds) throws InterruptedException {
+ if (monitorState[0] < actionCompletions.length) {
+ sleepsAndWaits.add("sleep:" + durationMilliseconds);
+ }
+ }
+ };
+
+ ActionExecutionInactivityWatchdog watchdog =
+ new ActionExecutionInactivityWatchdog(monitor, reporter, 0, sleep);
+ try {
+ synchronized (monitorFinishedIndicator) {
+ watchdog.start();
+
+ long startTime = System.currentTimeMillis();
+ boolean done = false;
+ while (!done) {
+ try {
+ monitorFinishedIndicator.wait(5000);
+ done = true;
+ MoreAsserts.assertLessThan(
+ "test didn't finish under 5 seconds",
+ 5000L,
+ System.currentTimeMillis() - startTime);
+ } catch (InterruptedException ie) {
+ // so-called Spurious Wakeup; ignore
+ }
+ }
+ }
+ } finally {
+ watchdog.stop();
+ }
+
+ assertEquals(shouldReport, didReportInactivity[0]);
+ assertThat(sleepsAndWaits)
+ .containsExactly(
+ "wait:5",
+ "sleep:1000",
+ "wait:10",
+ "wait:3",
+ "sleep:1000",
+ "wait:10",
+ "wait:30",
+ "wait:60",
+ "wait:60",
+ "wait:1")
+ .inOrder();
+ }
+
+ public void testInactivityWatchdogReportsWhenItShould() throws Exception {
+ assertInactivityWatchdogReports(true);
+ }
+
+ public void testInactivityWatchdogDoesNotReportWhenItShouldNot() throws Exception {
+ assertInactivityWatchdogReports(false);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java
new file mode 100644
index 0000000..e9efdfd
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java
@@ -0,0 +1,417 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.lib.skyframe.FileArtifactValue.create;
+import static org.junit.Assert.assertArrayEquals;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.testing.EqualsTester;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Action.MiddlemanType;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.MissingInputFileException;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.actions.util.TestAction.DummyAction;
+import com.google.devtools.build.lib.events.NullEventHandler;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.skyframe.ActionLookupValue.ActionLookupKey;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.RecordingDifferencer;
+import com.google.devtools.build.skyframe.SequentialBuildDriver;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Tests for {@link ArtifactFunction}.
+ */
+// Doesn't actually need any particular Skyframe, but is only relevant to Skyframe full mode.
+public class ArtifactFunctionTest extends TestCase {
+ private static final SkyKey OWNER_KEY = new SkyKey(SkyFunctions.ACTION_LOOKUP, "OWNER");
+ private static final ActionLookupKey ALL_OWNER = new SingletonActionLookupKey();
+
+ private Set<Action> actions;
+ private boolean fastDigest = false;
+ private RecordingDifferencer differencer = new RecordingDifferencer();
+ private SequentialBuildDriver driver;
+ private MemoizingEvaluator evaluator;
+ private Path root;
+ private TimestampGranularityMonitor tsgm = new TimestampGranularityMonitor(BlazeClock.instance());
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ setupRoot(new CustomInMemoryFs());
+ AtomicReference<PathPackageLocator> pkgLocator =
+ new AtomicReference<>(new PathPackageLocator());
+ ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator);
+ differencer = new RecordingDifferencer();
+ evaluator =
+ new InMemoryMemoizingEvaluator(
+ ImmutableMap.of(
+ SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper),
+ SkyFunctions.FILE, new FileFunction(pkgLocator, tsgm, externalFilesHelper),
+ SkyFunctions.ARTIFACT, new ArtifactFunction(Predicates.<PathFragment>alwaysFalse()),
+ SkyFunctions.ACTION_EXECUTION, new SimpleActionExecutionFunction()),
+ differencer);
+ driver = new SequentialBuildDriver(evaluator);
+ PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
+ actions = new HashSet<>();
+ }
+
+ private void setupRoot(CustomInMemoryFs fs) {
+ root = fs.getPath(TestUtils.tmpDir());
+ }
+
+ private void assertFileArtifactValueMatches(boolean expectDigest) throws Throwable {
+ Artifact output = createDerivedArtifact("output");
+ Path path = output.getPath();
+ file(path, "contents");
+ assertValueMatches(path.stat(), expectDigest ? path.getMD5Digest() : null, evaluateFAN(output));
+ }
+
+ public void testBasicArtifact() throws Throwable {
+ fastDigest = false;
+ assertFileArtifactValueMatches(/*expectDigest=*/ true);
+ }
+
+ public void testBasicArtifactWithXattr() throws Throwable {
+ fastDigest = true;
+ assertFileArtifactValueMatches(/*expectDigest=*/ true);
+ }
+
+ public void testMissingNonMandatoryArtifact() throws Throwable {
+ Artifact input = createSourceArtifact("input1");
+ assertNotNull(evaluateArtifactValue(input, /*mandatory=*/ false));
+ }
+
+ public void testMissingMandatoryArtifact() throws Throwable {
+ Artifact input = createSourceArtifact("input1");
+ try {
+ evaluateArtifactValue(input, /*mandatory=*/ true);
+ fail();
+ } catch (MissingInputFileException ex) {
+ // Expected.
+ }
+ }
+
+ public void testMiddlemanArtifact() throws Throwable {
+ Artifact output = createDerivedArtifact("output");
+ Artifact input1 = createSourceArtifact("input1");
+ Artifact input2 = createDerivedArtifact("input2");
+ Action action =
+ new DummyAction(
+ ImmutableList.of(input1, input2), output, MiddlemanType.AGGREGATING_MIDDLEMAN);
+ // Overwrite default generating action with this one.
+ for (Iterator<Action> it = actions.iterator(); it.hasNext(); ) {
+ if (it.next().getOutputs().contains(output)) {
+ it.remove();
+ break;
+ }
+ }
+ actions.add(action);
+ file(input2.getPath(), "contents");
+ file(input1.getPath(), "source contents");
+ evaluate(
+ Iterables.toArray(
+ ArtifactValue.mandatoryKeys(ImmutableSet.of(input2, input1, input2)), SkyKey.class));
+ ArtifactValue value = evaluateArtifactValue(output);
+ assertThat(((AggregatingArtifactValue) value).getInputs())
+ .containsExactly(Pair.of(input1, create(input1)), Pair.of(input2, create(input2)));
+ }
+
+ public void testIOException() throws Exception {
+ fastDigest = false;
+ final IOException exception = new IOException("beep");
+ setupRoot(
+ new CustomInMemoryFs() {
+ @Override
+ public byte[] getMD5Digest(Path path) throws IOException {
+ throw exception;
+ }
+ });
+ Artifact artifact = createDerivedArtifact("no-read");
+ writeFile(artifact.getPath(), "content");
+ try {
+ create(createDerivedArtifact("no-read"));
+ fail();
+ } catch (IOException e) {
+ assertSame(exception, e);
+ }
+ }
+
+ /**
+ * Tests that ArtifactFunction rethrows transitive {@link IOException}s as
+ * {@link MissingInputFileException}s.
+ */
+ public void testIOException_EndToEnd() throws Throwable {
+ final IOException exception = new IOException("beep");
+ setupRoot(
+ new CustomInMemoryFs() {
+ @Override
+ public FileStatus stat(Path path, boolean followSymlinks) throws IOException {
+ if (path.getBaseName().equals("bad")) {
+ throw exception;
+ }
+ return super.stat(path, followSymlinks);
+ }
+ });
+ try {
+ evaluateArtifactValue(createSourceArtifact("bad"));
+ fail();
+ } catch (MissingInputFileException e) {
+ assertThat(e.getMessage()).contains(exception.getMessage());
+ }
+ }
+
+ public void testNoMtimeIfNonemptyFile() throws Exception {
+ Artifact artifact = createDerivedArtifact("no-digest");
+ Path path = artifact.getPath();
+ writeFile(path, "hello"); //Non-empty file.
+ FileArtifactValue value = create(artifact);
+ assertArrayEquals(path.getMD5Digest(), value.getDigest());
+ try {
+ value.getModifiedTime();
+ fail("mtime for non-empty file should not be stored.");
+ } catch (IllegalStateException e) {
+ // Expected.
+ }
+ }
+
+ public void testDirectory() throws Exception {
+ Artifact artifact = createDerivedArtifact("dir");
+ Path path = artifact.getPath();
+ FileSystemUtils.createDirectoryAndParents(path);
+ path.setLastModifiedTime(1L);
+ FileArtifactValue value = create(artifact);
+ assertNull(value.getDigest());
+ assertEquals(1L, value.getModifiedTime());
+ }
+
+ // Empty files need to store their mtimes, so touching an empty file
+ // can be used to trigger rebuilds.
+ public void testEmptyFile() throws Exception {
+ Artifact artifact = createDerivedArtifact("empty");
+ Path path = artifact.getPath();
+ writeFile(path, "");
+ path.setLastModifiedTime(1L);
+ FileArtifactValue value = create(artifact);
+ assertArrayEquals(path.getMD5Digest(), value.getDigest());
+ assertEquals(1L, value.getModifiedTime());
+ assertEquals(0L, value.getSize());
+ }
+
+ public void testEquality() throws Exception {
+ Artifact artifact1 = createDerivedArtifact("artifact1");
+ Artifact artifact2 = createDerivedArtifact("artifact2");
+ Artifact diffDigest = createDerivedArtifact("diffDigest");
+ Artifact diffMtime = createDerivedArtifact("diffMtime");
+ Artifact empty1 = createDerivedArtifact("empty1");
+ Artifact empty2 = createDerivedArtifact("empty2");
+ Artifact empty3 = createDerivedArtifact("empty3");
+ Artifact dir1 = createDerivedArtifact("dir1");
+ Artifact dir2 = createDerivedArtifact("dir2");
+ Artifact dir3 = createDerivedArtifact("dir3");
+ Path path1 = artifact1.getPath();
+ Path path2 = artifact2.getPath();
+ Path digestPath = diffDigest.getPath();
+ Path mtimePath = diffMtime.getPath();
+ writeFile(artifact1.getPath(), "content");
+ writeFile(artifact2.getPath(), "content");
+ path1.setLastModifiedTime(0);
+ path2.setLastModifiedTime(0);
+ writeFile(diffDigest.getPath(), "1234567"); // Same size as artifact1.
+ digestPath.setLastModifiedTime(0);
+ writeFile(mtimePath, "content");
+ mtimePath.setLastModifiedTime(1);
+ Path emptyPath1 = empty1.getPath();
+ Path emptyPath2 = empty2.getPath();
+ Path emptyPath3 = empty3.getPath();
+ writeFile(emptyPath1, "");
+ writeFile(emptyPath2, "");
+ writeFile(emptyPath3, "");
+ emptyPath1.setLastModifiedTime(0L);
+ emptyPath2.setLastModifiedTime(1L);
+ emptyPath3.setLastModifiedTime(1L);
+ Path dirPath1 = dir1.getPath();
+ Path dirPath2 = dir2.getPath();
+ Path dirPath3 = dir3.getPath();
+ FileSystemUtils.createDirectoryAndParents(dirPath1);
+ FileSystemUtils.createDirectoryAndParents(dirPath2);
+ FileSystemUtils.createDirectoryAndParents(dirPath3);
+ dirPath1.setLastModifiedTime(0L);
+ dirPath2.setLastModifiedTime(1L);
+ dirPath3.setLastModifiedTime(1L);
+ EqualsTester equalsTester = new EqualsTester();
+ equalsTester
+ .addEqualityGroup(create(artifact1), create(artifact2), create(diffMtime))
+ .addEqualityGroup(create(empty1))
+ .addEqualityGroup(create(empty2), create(empty3))
+ .addEqualityGroup(create(dir1))
+ .addEqualityGroup(create(dir2), create(dir3))
+ .testEquals();
+ }
+
+ private void file(Path path, String contents) throws Exception {
+ FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
+ writeFile(path, contents);
+ }
+
+ private Artifact createSourceArtifact(String path) {
+ return new Artifact(new PathFragment(path), Root.asSourceRoot(root));
+ }
+
+ private Artifact createDerivedArtifact(String path) {
+ PathFragment execPath = new PathFragment("out").getRelative(path);
+ Path fullPath = root.getRelative(execPath);
+ Artifact output =
+ new Artifact(
+ fullPath, Root.asDerivedRoot(root, root.getRelative("out")), execPath, ALL_OWNER);
+ actions.add(new DummyAction(ImmutableList.<Artifact>of(), output));
+ return output;
+ }
+
+ private void assertValueMatches(FileStatus file, byte[] digest, FileArtifactValue value)
+ throws IOException {
+ assertEquals(file.getSize(), value.getSize());
+ if (digest == null) {
+ assertNull(value.getDigest());
+ assertEquals(file.getLastModifiedTime(), value.getModifiedTime());
+ } else {
+ assertArrayEquals(digest, value.getDigest());
+ }
+ }
+
+ private FileArtifactValue evaluateFAN(Artifact artifact) throws Throwable {
+ return ((FileArtifactValue) evaluateArtifactValue(artifact));
+ }
+
+ private ArtifactValue evaluateArtifactValue(Artifact artifact) throws Throwable {
+ return evaluateArtifactValue(artifact, /*isMandatory=*/ true);
+ }
+
+ private ArtifactValue evaluateArtifactValue(Artifact artifact, boolean mandatory)
+ throws Throwable {
+ SkyKey key = ArtifactValue.key(artifact, mandatory);
+ EvaluationResult<ArtifactValue> result = evaluate(ImmutableList.of(key).toArray(new SkyKey[0]));
+ if (result.hasError()) {
+ throw result.getError().getException();
+ }
+ return result.get(key);
+ }
+
+ private void setGeneratingActions() {
+ if (evaluator.getExistingValueForTesting(OWNER_KEY) == null) {
+ differencer.inject(ImmutableMap.of(OWNER_KEY, new ActionLookupValue(actions)));
+ }
+ }
+
+ private <E extends SkyValue> EvaluationResult<E> evaluate(SkyKey... keys)
+ throws InterruptedException {
+ setGeneratingActions();
+ return driver.evaluate(
+ Arrays.asList(keys), /*keepGoing=*/
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ }
+
+ private static void writeFile(Path path, String contents) throws IOException {
+ FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
+ FileSystemUtils.writeContentAsLatin1(path, contents);
+ }
+
+ private static class SingletonActionLookupKey extends ActionLookupKey {
+ @Override
+ SkyKey getSkyKey() {
+ return OWNER_KEY;
+ }
+
+ @Override
+ SkyFunctionName getType() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ /** Value Builder for actions that just stats and stores the output file (which must exist). */
+ private class SimpleActionExecutionFunction implements SkyFunction {
+ @Override
+ public SkyValue compute(SkyKey skyKey, Environment env) {
+ Map<Artifact, FileValue> artifactData = new HashMap<>();
+ Action action = (Action) skyKey.argument();
+ Artifact output = Iterables.getOnlyElement(action.getOutputs());
+ FileArtifactValue value;
+ if (action.getActionType() == MiddlemanType.NORMAL) {
+ try {
+ FileValue fileValue = ActionMetadataHandler.fileValueFromArtifact(output, null, tsgm);
+ artifactData.put(output, fileValue);
+ value = FileArtifactValue.create(output, fileValue);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ } else {
+ value = FileArtifactValue.DEFAULT_MIDDLEMAN;
+ }
+ return new ActionExecutionValue(artifactData, ImmutableMap.of(output, value));
+ }
+
+ @Override
+ public String extractTag(SkyKey skyKey) {
+ return null;
+ }
+ }
+
+ /** InMemoryFileSystem that can pretend to do a fast digest. */
+ private class CustomInMemoryFs extends InMemoryFileSystem {
+ @Override
+ protected String getFastDigestFunctionType(Path path) {
+ return fastDigest ? "MD5" : null;
+ }
+
+ @Override
+ protected byte[] getFastDigest(Path path) throws IOException {
+ return fastDigest ? getMD5Digest(path) : null;
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunctionTest.java
new file mode 100644
index 0000000..c7c7d5b
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/ContainingPackageLookupFunctionTest.java
@@ -0,0 +1,126 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.testing.EqualsTester;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.events.NullEventHandler;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.testutil.FoundationTestCase;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.RecordingDifferencer;
+import com.google.devtools.build.skyframe.SequentialBuildDriver;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Tests for {@link ContainingPackageLookupFunction}.
+ */
+public class ContainingPackageLookupFunctionTest extends FoundationTestCase {
+
+ private AtomicReference<ImmutableSet<PackageIdentifier>> deletedPackages;
+ private MemoizingEvaluator evaluator;
+ private SequentialBuildDriver driver;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ AtomicReference<PathPackageLocator> pkgLocator =
+ new AtomicReference<>(new PathPackageLocator(rootDirectory));
+ deletedPackages = new AtomicReference<>(ImmutableSet.<PackageIdentifier>of());
+ ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator);
+ TimestampGranularityMonitor tsgm = new TimestampGranularityMonitor(BlazeClock.instance());
+
+ Map<SkyFunctionName, SkyFunction> skyFunctions = new HashMap<>();
+ skyFunctions.put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(deletedPackages));
+ skyFunctions.put(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, new ContainingPackageLookupFunction());
+ skyFunctions.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
+ skyFunctions.put(SkyFunctions.FILE, new FileFunction(pkgLocator, tsgm, externalFilesHelper));
+ RecordingDifferencer differencer = new RecordingDifferencer();
+ evaluator = new InMemoryMemoizingEvaluator(skyFunctions, differencer);
+ driver = new SequentialBuildDriver(evaluator);
+ PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
+ PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
+ }
+
+ private ContainingPackageLookupValue lookupContainingPackage(String packageName)
+ throws InterruptedException {
+ SkyKey key =
+ ContainingPackageLookupValue.key(PackageIdentifier.createInDefaultRepo(packageName));
+ return driver
+ .<ContainingPackageLookupValue>evaluate(
+ ImmutableList.of(key),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE)
+ .get(key);
+ }
+
+ public void testNoContainingPackage() throws Exception {
+ ContainingPackageLookupValue value = lookupContainingPackage("a/b");
+ assertFalse(value.hasContainingPackage());
+ }
+
+ public void testContainingPackageIsParent() throws Exception {
+ scratch.file("a/BUILD");
+ ContainingPackageLookupValue value = lookupContainingPackage("a/b");
+ assertTrue(value.hasContainingPackage());
+ assertEquals(PackageIdentifier.createInDefaultRepo("a"), value.getContainingPackageName());
+ assertEquals(rootDirectory, value.getContainingPackageRoot());
+ }
+
+ public void testContainingPackageIsSelf() throws Exception {
+ scratch.file("a/b/BUILD");
+ ContainingPackageLookupValue value = lookupContainingPackage("a/b");
+ assertTrue(value.hasContainingPackage());
+ assertEquals(PackageIdentifier.createInDefaultRepo("a/b"), value.getContainingPackageName());
+ assertEquals(rootDirectory, value.getContainingPackageRoot());
+ }
+
+ public void testEqualsAndHashCodeContract() throws Exception {
+ ContainingPackageLookupValue valueA1 = ContainingPackageLookupValue.NONE;
+ ContainingPackageLookupValue valueA2 = ContainingPackageLookupValue.NONE;
+ ContainingPackageLookupValue valueB1 =
+ ContainingPackageLookupValue.withContainingPackage(
+ PackageIdentifier.createInDefaultRepo("b"), rootDirectory);
+ ContainingPackageLookupValue valueB2 =
+ ContainingPackageLookupValue.withContainingPackage(
+ PackageIdentifier.createInDefaultRepo("b"), rootDirectory);
+ PackageIdentifier cFrag = PackageIdentifier.createInDefaultRepo("c");
+ ContainingPackageLookupValue valueC1 =
+ ContainingPackageLookupValue.withContainingPackage(cFrag, rootDirectory);
+ ContainingPackageLookupValue valueC2 =
+ ContainingPackageLookupValue.withContainingPackage(cFrag, rootDirectory);
+ ContainingPackageLookupValue valueCOther =
+ ContainingPackageLookupValue.withContainingPackage(
+ cFrag, rootDirectory.getRelative("other_root"));
+ new EqualsTester()
+ .addEqualityGroup(valueA1, valueA2)
+ .addEqualityGroup(valueB1, valueB2)
+ .addEqualityGroup(valueC1, valueC2)
+ .addEqualityGroup(valueCOther)
+ .testEquals();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManagerTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManagerTest.java
new file mode 100644
index 0000000..bd28c33
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/DiffAwarenessManagerTest.java
@@ -0,0 +1,266 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.events.util.EventCollectionApparatus;
+import com.google.devtools.build.lib.skyframe.DiffAwarenessManager.ProcessableModifiedFileSet;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import junit.framework.TestCase;
+
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+/**
+ * Unit tests for {@link DiffAwarenessManager}, especially of the fact that it works in a sequential
+ * manner and of its correctness in the presence of unprocesed diffs.
+ */
+public class DiffAwarenessManagerTest extends TestCase {
+
+ private FileSystem fs;
+ private Path root;
+ protected EventCollectionApparatus events;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ fs = new InMemoryFileSystem();
+ root = fs.getRootDirectory();
+ events = new EventCollectionApparatus();
+ events.setFailFast(false);
+ }
+
+ public void testEverythingModifiedIfNoDiffAwareness() throws Exception {
+ Path pathEntry = root.getRelative("pathEntry");
+ DiffAwarenessFactoryStub factory = new DiffAwarenessFactoryStub();
+ DiffAwarenessManager manager = new DiffAwarenessManager(ImmutableList.of(factory));
+ assertEquals(
+ "Expected EVERYTHING_MODIFIED since there are no factories",
+ ModifiedFileSet.EVERYTHING_MODIFIED,
+ manager.getDiff(events.reporter(), pathEntry).getModifiedFileSet());
+ events.assertNoWarningsOrErrors();
+ }
+
+ public void testResetAndSetPathEntriesCallClose() throws Exception {
+ Path pathEntry = root.getRelative("pathEntry");
+ ModifiedFileSet diff = ModifiedFileSet.NOTHING_MODIFIED;
+ DiffAwarenessStub diffAwareness1 = new DiffAwarenessStub(ImmutableList.of(diff));
+ DiffAwarenessStub diffAwareness2 = new DiffAwarenessStub(ImmutableList.of(diff));
+ DiffAwarenessFactoryStub factory = new DiffAwarenessFactoryStub();
+ factory.inject(pathEntry, diffAwareness1);
+ DiffAwarenessManager manager = new DiffAwarenessManager(ImmutableList.of(factory));
+ manager.getDiff(events.reporter(), pathEntry);
+ assertFalse("diffAwareness1 shouldn't have been closed yet", diffAwareness1.closed());
+ manager.reset();
+ assertTrue("diffAwareness1 should have been closed by reset", diffAwareness1.closed());
+ factory.inject(pathEntry, diffAwareness2);
+ manager.getDiff(events.reporter(), pathEntry);
+ assertFalse("diffAwareness2 shouldn't have been closed yet", diffAwareness2.closed());
+ events.assertNoWarningsOrErrors();
+ }
+
+ public void testHandlesUnprocessedDiffs() throws Exception {
+ Path pathEntry = root.getRelative("pathEntry");
+ ModifiedFileSet diff1 = ModifiedFileSet.builder().modify(new PathFragment("file1")).build();
+ ModifiedFileSet diff2 = ModifiedFileSet.builder().modify(new PathFragment("file2")).build();
+ ModifiedFileSet diff3 = ModifiedFileSet.builder().modify(new PathFragment("file3")).build();
+ DiffAwarenessStub diffAwareness =
+ new DiffAwarenessStub(ImmutableList.of(diff1, diff2, diff3, DiffAwarenessStub.BROKEN_DIFF));
+ DiffAwarenessFactoryStub factory = new DiffAwarenessFactoryStub();
+ factory.inject(pathEntry, diffAwareness);
+ DiffAwarenessManager manager = new DiffAwarenessManager(ImmutableList.of(factory));
+ ProcessableModifiedFileSet firstProcessableDiff = manager.getDiff(events.reporter(), pathEntry);
+ assertEquals(
+ "Expected EVERYTHING_MODIFIED on first call to getDiff",
+ ModifiedFileSet.EVERYTHING_MODIFIED,
+ firstProcessableDiff.getModifiedFileSet());
+ firstProcessableDiff.markProcessed();
+ ProcessableModifiedFileSet processableDiff1 = manager.getDiff(events.reporter(), pathEntry);
+ assertEquals(diff1, processableDiff1.getModifiedFileSet());
+ ProcessableModifiedFileSet processableDiff2 = manager.getDiff(events.reporter(), pathEntry);
+ assertEquals(ModifiedFileSet.union(diff1, diff2), processableDiff2.getModifiedFileSet());
+ processableDiff2.markProcessed();
+ ProcessableModifiedFileSet processableDiff3 = manager.getDiff(events.reporter(), pathEntry);
+ assertEquals(diff3, processableDiff3.getModifiedFileSet());
+ events.assertNoWarningsOrErrors();
+ ProcessableModifiedFileSet processableDiff4 = manager.getDiff(events.reporter(), pathEntry);
+ assertEquals(ModifiedFileSet.EVERYTHING_MODIFIED, processableDiff4.getModifiedFileSet());
+ events.assertContainsWarning("error");
+ }
+
+ public void testHandlesBrokenDiffs() throws Exception {
+ Path pathEntry = root.getRelative("pathEntry");
+ DiffAwarenessFactoryStub factory1 = new DiffAwarenessFactoryStub();
+ DiffAwarenessStub diffAwareness1 =
+ new DiffAwarenessStub(ImmutableList.<ModifiedFileSet>of(), 1);
+ factory1.inject(pathEntry, diffAwareness1);
+ DiffAwarenessFactoryStub factory2 = new DiffAwarenessFactoryStub();
+ ModifiedFileSet diff2 = ModifiedFileSet.builder().modify(new PathFragment("file2")).build();
+ DiffAwarenessStub diffAwareness2 =
+ new DiffAwarenessStub(ImmutableList.of(diff2, DiffAwarenessStub.BROKEN_DIFF));
+ factory2.inject(pathEntry, diffAwareness2);
+ DiffAwarenessFactoryStub factory3 = new DiffAwarenessFactoryStub();
+ ModifiedFileSet diff3 = ModifiedFileSet.builder().modify(new PathFragment("file3")).build();
+ DiffAwarenessStub diffAwareness3 = new DiffAwarenessStub(ImmutableList.of(diff3));
+ factory3.inject(pathEntry, diffAwareness3);
+ DiffAwarenessManager manager =
+ new DiffAwarenessManager(ImmutableList.of(factory1, factory2, factory3));
+
+ ProcessableModifiedFileSet processableDiff = manager.getDiff(events.reporter(), pathEntry);
+ events.assertNoWarningsOrErrors();
+ assertEquals(
+ "Expected EVERYTHING_MODIFIED on first call to getDiff for diffAwareness1",
+ ModifiedFileSet.EVERYTHING_MODIFIED,
+ processableDiff.getModifiedFileSet());
+ processableDiff.markProcessed();
+
+ processableDiff = manager.getDiff(events.reporter(), pathEntry);
+ events.assertContainsEventWithFrequency("error in getCurrentView", 1);
+ assertEquals(
+ "Expected EVERYTHING_MODIFIED because of broken getCurrentView",
+ ModifiedFileSet.EVERYTHING_MODIFIED,
+ processableDiff.getModifiedFileSet());
+ processableDiff.markProcessed();
+ factory1.remove(pathEntry);
+
+ processableDiff = manager.getDiff(events.reporter(), pathEntry);
+ assertEquals(
+ "Expected EVERYTHING_MODIFIED on first call to getDiff for diffAwareness2",
+ ModifiedFileSet.EVERYTHING_MODIFIED,
+ processableDiff.getModifiedFileSet());
+ processableDiff.markProcessed();
+
+ processableDiff = manager.getDiff(events.reporter(), pathEntry);
+ assertEquals(diff2, processableDiff.getModifiedFileSet());
+ processableDiff.markProcessed();
+
+ processableDiff = manager.getDiff(events.reporter(), pathEntry);
+ events.assertContainsEventWithFrequency("error in getDiff", 1);
+ assertEquals(
+ "Expected EVERYTHING_MODIFIED because of broken getDiff",
+ ModifiedFileSet.EVERYTHING_MODIFIED,
+ processableDiff.getModifiedFileSet());
+ processableDiff.markProcessed();
+ factory2.remove(pathEntry);
+
+ processableDiff = manager.getDiff(events.reporter(), pathEntry);
+ assertEquals(
+ "Expected EVERYTHING_MODIFIED on first call to getDiff for diffAwareness3",
+ ModifiedFileSet.EVERYTHING_MODIFIED,
+ processableDiff.getModifiedFileSet());
+ processableDiff.markProcessed();
+
+ processableDiff = manager.getDiff(events.reporter(), pathEntry);
+ assertEquals(diff3, processableDiff.getModifiedFileSet());
+ processableDiff.markProcessed();
+ }
+
+ private static class DiffAwarenessFactoryStub implements DiffAwareness.Factory {
+
+ private Map<Path, DiffAwareness> diffAwarenesses = Maps.newHashMap();
+
+ public void inject(Path pathEntry, DiffAwareness diffAwareness) {
+ diffAwarenesses.put(pathEntry, diffAwareness);
+ }
+
+ public void remove(Path pathEntry) {
+ diffAwarenesses.remove(pathEntry);
+ }
+
+ @Override
+ @Nullable
+ public DiffAwareness maybeCreate(Path pathEntry) {
+ return diffAwarenesses.get(pathEntry);
+ }
+ }
+
+ private static class DiffAwarenessStub implements DiffAwareness {
+
+ public static final ModifiedFileSet BROKEN_DIFF =
+ ModifiedFileSet.builder().modify(new PathFragment("special broken marker")).build();
+
+ private boolean closed = false;
+ private int curSequenceNum = 0;
+ private final List<ModifiedFileSet> sequentialDiffs;
+ private final int brokenViewNum;
+
+ public DiffAwarenessStub(List<ModifiedFileSet> sequentialDiffs) {
+ this(sequentialDiffs, -1);
+ }
+
+ public DiffAwarenessStub(List<ModifiedFileSet> sequentialDiffs, int brokenViewNum) {
+ this.sequentialDiffs = sequentialDiffs;
+ this.brokenViewNum = brokenViewNum;
+ }
+
+ private static class ViewStub implements DiffAwareness.View {
+ private final int sequenceNum;
+
+ public ViewStub(int sequenceNum) {
+ this.sequenceNum = sequenceNum;
+ }
+ }
+
+ @Override
+ public View getCurrentView() throws BrokenDiffAwarenessException {
+ if (curSequenceNum == brokenViewNum) {
+ throw new BrokenDiffAwarenessException("error in getCurrentView");
+ }
+ return new ViewStub(curSequenceNum++);
+ }
+
+ @Override
+ public ModifiedFileSet getDiff(View oldView, View newView) throws BrokenDiffAwarenessException {
+ assertThat(oldView).isInstanceOf(ViewStub.class);
+ assertThat(newView).isInstanceOf(ViewStub.class);
+ ViewStub oldViewStub = (ViewStub) oldView;
+ ViewStub newViewStub = (ViewStub) newView;
+ Preconditions.checkState(newViewStub.sequenceNum >= oldViewStub.sequenceNum);
+ ModifiedFileSet diff = ModifiedFileSet.NOTHING_MODIFIED;
+ for (int num = oldViewStub.sequenceNum; num < newViewStub.sequenceNum; num++) {
+ ModifiedFileSet incrementalDiff = sequentialDiffs.get(num);
+ if (incrementalDiff == BROKEN_DIFF) {
+ throw new BrokenDiffAwarenessException("error in getDiff");
+ }
+ diff = ModifiedFileSet.union(diff, incrementalDiff);
+ }
+ return diff;
+ }
+
+ @Override
+ public String name() {
+ return "testingstub";
+ }
+
+ @Override
+ public void close() {
+ closed = true;
+ }
+
+ public boolean closed() {
+ return closed;
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunctionTest.java
new file mode 100644
index 0000000..edfc2c5
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/FileSymlinkCycleUniquenessFunctionTest.java
@@ -0,0 +1,46 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.testing.EqualsTester;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+
+import junit.framework.TestCase;
+
+public class FileSymlinkCycleUniquenessFunctionTest extends TestCase {
+
+ public void testHashCodeAndEqualsContract() throws Exception {
+ Path root = new InMemoryFileSystem().getRootDirectory().getRelative("root");
+ RootedPath p1 = RootedPath.toRootedPath(root, new PathFragment("p1"));
+ RootedPath p2 = RootedPath.toRootedPath(root, new PathFragment("p2"));
+ RootedPath p3 = RootedPath.toRootedPath(root, new PathFragment("p3"));
+ ImmutableList<RootedPath> cycleA1 = ImmutableList.of(p1);
+ ImmutableList<RootedPath> cycleB1 = ImmutableList.of(p2);
+ ImmutableList<RootedPath> cycleC1 = ImmutableList.of(p1, p2, p3);
+ ImmutableList<RootedPath> cycleC2 = ImmutableList.of(p2, p3, p1);
+ ImmutableList<RootedPath> cycleC3 = ImmutableList.of(p3, p1, p2);
+ new EqualsTester()
+ .addEqualityGroup(FileSymlinkCycleUniquenessFunction.key(cycleA1))
+ .addEqualityGroup(FileSymlinkCycleUniquenessFunction.key(cycleB1))
+ .addEqualityGroup(
+ FileSymlinkCycleUniquenessFunction.key(cycleC1),
+ FileSymlinkCycleUniquenessFunction.key(cycleC2),
+ FileSymlinkCycleUniquenessFunction.key(cycleC3))
+ .testEquals();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunctionTest.java
new file mode 100644
index 0000000..e699f0a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/FilesetEntryFunctionTest.java
@@ -0,0 +1,853 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static com.google.devtools.build.lib.actions.FilesetTraversalParams.PackageBoundaryMode.CROSS;
+import static com.google.devtools.build.lib.actions.FilesetTraversalParams.PackageBoundaryMode.DONT_CROSS;
+import static com.google.devtools.build.lib.actions.FilesetTraversalParams.PackageBoundaryMode.REPORT_ERROR;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
+import com.google.devtools.build.lib.actions.FilesetTraversalParams;
+import com.google.devtools.build.lib.actions.FilesetTraversalParams.PackageBoundaryMode;
+import com.google.devtools.build.lib.actions.FilesetTraversalParamsFactory;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.events.NullEventHandler;
+import com.google.devtools.build.lib.packages.FilesetEntry.SymlinkBehavior;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.testutil.FoundationTestCase;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.RecordingDifferencer;
+import com.google.devtools.build.skyframe.SequentialBuildDriver;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/** Tests for {@link FilesetEntryFunction}. */
+public final class FilesetEntryFunctionTest extends FoundationTestCase {
+
+ private TimestampGranularityMonitor tsgm = new TimestampGranularityMonitor(BlazeClock.instance());
+ private MemoizingEvaluator evaluator;
+ private SequentialBuildDriver driver;
+ private RecordingDifferencer differencer;
+ private AtomicReference<PathPackageLocator> pkgLocator;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+
+ pkgLocator = new AtomicReference<>(new PathPackageLocator(rootDirectory));
+ AtomicReference<ImmutableSet<PackageIdentifier>> deletedPackages =
+ new AtomicReference<>(ImmutableSet.<PackageIdentifier>of());
+ ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator);
+
+ Map<SkyFunctionName, SkyFunction> skyFunctions = new HashMap<>();
+
+ skyFunctions.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
+ skyFunctions.put(SkyFunctions.FILE, new FileFunction(pkgLocator, tsgm, externalFilesHelper));
+ skyFunctions.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction());
+ skyFunctions.put(
+ SkyFunctions.DIRECTORY_LISTING_STATE,
+ new DirectoryListingStateFunction(externalFilesHelper));
+ skyFunctions.put(
+ SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, new RecursiveFilesystemTraversalFunction());
+ skyFunctions.put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(deletedPackages));
+ skyFunctions.put(SkyFunctions.FILESET_ENTRY, new FilesetEntryFunction());
+
+ differencer = new RecordingDifferencer();
+ evaluator = new InMemoryMemoizingEvaluator(skyFunctions, differencer);
+ driver = new SequentialBuildDriver(evaluator);
+ PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
+ PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
+ }
+
+ private Artifact getSourceArtifact(String path) throws Exception {
+ return new Artifact(new PathFragment(path), Root.asSourceRoot(rootDirectory));
+ }
+
+ private Artifact createSourceArtifact(String path) throws Exception {
+ Artifact result = getSourceArtifact(path);
+ createFile(result, "foo");
+ return result;
+ }
+
+ private static RootedPath rootedPath(Artifact artifact) {
+ return RootedPath.toRootedPath(artifact.getRoot().getPath(), artifact.getRootRelativePath());
+ }
+
+ private static RootedPath childOf(Artifact artifact, String relative) {
+ return RootedPath.toRootedPath(
+ artifact.getRoot().getPath(), artifact.getRootRelativePath().getRelative(relative));
+ }
+
+ private static RootedPath siblingOf(Artifact artifact, String relative) {
+ PathFragment parent =
+ Preconditions.checkNotNull(artifact.getRootRelativePath().getParentDirectory());
+ return RootedPath.toRootedPath(artifact.getRoot().getPath(), parent.getRelative(relative));
+ }
+
+ private void createFile(Path path, String... contents) throws Exception {
+ if (!path.getParentDirectory().exists()) {
+ scratch.dir(path.getParentDirectory().getPathString());
+ }
+ scratch.file(path.getPathString(), contents);
+ }
+
+ private void createFile(Artifact artifact, String... contents) throws Exception {
+ createFile(artifact.getPath(), contents);
+ }
+
+ private RootedPath createFile(RootedPath path, String... contents) throws Exception {
+ createFile(path.asPath(), contents);
+ return path;
+ }
+
+ private <T extends SkyValue> EvaluationResult<T> eval(SkyKey key) throws Exception {
+ return driver.evaluate(
+ ImmutableList.of(key),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ }
+
+ private FilesetEntryValue evalFilesetTraversal(FilesetTraversalParams params) throws Exception {
+ SkyKey key = FilesetEntryValue.key(params);
+ EvaluationResult<FilesetEntryValue> result = eval(key);
+ assertThat(result.hasError()).isFalse();
+ return result.get(key);
+ }
+
+ private static FilesetOutputSymlink symlink(String from, Artifact to) {
+ return new FilesetOutputSymlink(new PathFragment(from), to.getPath().asFragment());
+ }
+
+ private static FilesetOutputSymlink symlink(String from, String to) {
+ return new FilesetOutputSymlink(new PathFragment(from), new PathFragment(to));
+ }
+
+ private static FilesetOutputSymlink symlink(String from, RootedPath to) {
+ return new FilesetOutputSymlink(new PathFragment(from), to.asPath().asFragment());
+ }
+
+ private void assertSymlinksInOrder(
+ FilesetTraversalParams request, FilesetOutputSymlink... expectedSymlinks) throws Exception {
+ List<FilesetOutputSymlink> expected = Arrays.asList(expectedSymlinks);
+ Collection<FilesetOutputSymlink> actual =
+ Collections2.transform(
+ evalFilesetTraversal(request).getSymlinks(),
+ // Strip the metadata from the actual results.
+ new Function<FilesetOutputSymlink, FilesetOutputSymlink>() {
+ @Override
+ public FilesetOutputSymlink apply(FilesetOutputSymlink input) {
+ return new FilesetOutputSymlink(input.name, input.target);
+ }
+ });
+ assertThat(actual).containsExactlyElementsIn(expected).inOrder();
+ }
+
+ private static Label label(String label) throws Exception {
+ return Label.parseAbsolute(label);
+ }
+
+ public void testFileTraversalForFile() throws Exception {
+ Artifact file = createSourceArtifact("foo/file.real");
+ FilesetTraversalParams params =
+ FilesetTraversalParamsFactory.fileTraversal(
+ /*ownerLabel=*/ label("//foo"),
+ /*fileToTraverse=*/ file,
+ /*destPath=*/ new PathFragment("output-name"),
+ /*symlinkBehaviorMode=*/ SymlinkBehavior.COPY,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+ assertSymlinksInOrder(params, symlink("output-name", file));
+ }
+
+ private void assertFileTraversalForFileSymlink(SymlinkBehavior symlinks) throws Exception {
+ Artifact file = createSourceArtifact("foo/file.real");
+ Artifact symlink = getSourceArtifact("foo/file.sym");
+ symlink.getPath().createSymbolicLink(new PathFragment("file.real"));
+
+ FilesetTraversalParams params =
+ FilesetTraversalParamsFactory.fileTraversal(
+ /*ownerLabel=*/ label("//foo"),
+ /*fileToTraverse=*/ symlink,
+ /*destPath=*/ new PathFragment("output-name"),
+ /*symlinkBehaviorMode=*/ symlinks,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+ switch (symlinks) {
+ case COPY:
+ assertSymlinksInOrder(params, symlink("output-name", "file.real"));
+ break;
+ case DEREFERENCE:
+ assertSymlinksInOrder(params, symlink("output-name", file));
+ break;
+ default:
+ throw new IllegalStateException(symlinks.toString());
+ }
+ }
+
+ public void testFileTraversalForFileSymlinkNoFollow() throws Exception {
+ assertFileTraversalForFileSymlink(SymlinkBehavior.COPY);
+ }
+
+ public void testFileTraversalForFileSymlinkFollow() throws Exception {
+ assertFileTraversalForFileSymlink(SymlinkBehavior.DEREFERENCE);
+ }
+
+ public void testFileTraversalForDirectory() throws Exception {
+ Artifact dir = getSourceArtifact("foo/dir_real");
+ RootedPath fileA = createFile(childOf(dir, "file.a"), "hello");
+ RootedPath fileB = createFile(childOf(dir, "sub/file.b"), "world");
+
+ FilesetTraversalParams params =
+ FilesetTraversalParamsFactory.fileTraversal(
+ /*ownerLabel=*/ label("//foo"),
+ /*fileToTraverse=*/ dir,
+ /*destPath=*/ new PathFragment("output-name"),
+ /*symlinkBehaviorMode=*/ SymlinkBehavior.COPY,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+ assertSymlinksInOrder(
+ params, symlink("output-name/file.a", fileA), symlink("output-name/sub/file.b", fileB));
+ }
+
+ private void assertFileTraversalForDirectorySymlink(SymlinkBehavior symlinks) throws Exception {
+ Artifact dir = getSourceArtifact("foo/dir_real");
+ Artifact symlink = getSourceArtifact("foo/dir_sym");
+ createFile(childOf(dir, "file.a"), "hello");
+ createFile(childOf(dir, "sub/file.b"), "world");
+ symlink.getPath().createSymbolicLink(new PathFragment("dir_real"));
+
+ FilesetTraversalParams params =
+ FilesetTraversalParamsFactory.fileTraversal(
+ /*ownerLabel=*/ label("//foo"),
+ /*fileToTraverse=*/ symlink,
+ /*destPath=*/ new PathFragment("output-name"),
+ /*symlinkBehaviorMode=*/ symlinks,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+ switch (symlinks) {
+ case COPY:
+ assertSymlinksInOrder(params, symlink("output-name", "dir_real"));
+ break;
+ case DEREFERENCE:
+ assertSymlinksInOrder(params, symlink("output-name", dir));
+ break;
+ default:
+ throw new IllegalStateException(symlinks.toString());
+ }
+ }
+
+ public void testFileTraversalForDirectorySymlinkFollow() throws Exception {
+ assertFileTraversalForDirectorySymlink(SymlinkBehavior.COPY);
+ }
+
+ public void testFileTraversalForDirectorySymlinkNoFollow() throws Exception {
+ assertFileTraversalForDirectorySymlink(SymlinkBehavior.DEREFERENCE);
+ }
+
+ private void assertRecursiveTraversalForDirectory(
+ SymlinkBehavior symlinks, PackageBoundaryMode pkgBoundaryMode) throws Exception {
+ Artifact dir = getSourceArtifact("foo/dir");
+ RootedPath fileA = createFile(childOf(dir, "file.a"), "blah");
+ RootedPath fileAsym = childOf(dir, "subdir/file.a.sym");
+ RootedPath buildFile = createFile(childOf(dir, "subpkg/BUILD"), "blah");
+ RootedPath fileB = createFile(childOf(dir, "subpkg/file.b"), "blah");
+ fileAsym.asPath().getParentDirectory().createDirectory();
+ fileAsym.asPath().createSymbolicLink(new PathFragment("../file.a"));
+
+ FilesetOutputSymlink outA = symlink("output-name/file.a", childOf(dir, "file.a"));
+ FilesetOutputSymlink outAsym = null;
+ FilesetOutputSymlink outBuild = symlink("output-name/subpkg/BUILD", buildFile);
+ FilesetOutputSymlink outB = symlink("output-name/subpkg/file.b", fileB);
+ switch (symlinks) {
+ case COPY:
+ outAsym = symlink("output-name/subdir/file.a.sym", "../file.a");
+ break;
+ case DEREFERENCE:
+ outAsym = symlink("output-name/subdir/file.a.sym", fileA);
+ break;
+ default:
+ throw new IllegalStateException(symlinks.toString());
+ }
+
+ FilesetTraversalParams params =
+ FilesetTraversalParamsFactory.recursiveTraversalOfDirectory(
+ /*ownerLabel=*/ label("//foo"),
+ /*directoryToTraverse=*/ dir,
+ /*destPath=*/ new PathFragment("output-name"),
+ /*excludes=*/ null,
+ /*symlinkBehaviorMode=*/ symlinks,
+ /*pkgBoundaryMode=*/ pkgBoundaryMode);
+ switch (pkgBoundaryMode) {
+ case CROSS:
+ assertSymlinksInOrder(params, outA, outAsym, outBuild, outB);
+ break;
+ case DONT_CROSS:
+ assertSymlinksInOrder(params, outA, outAsym);
+ break;
+ case REPORT_ERROR:
+ SkyKey key = FilesetEntryValue.key(params);
+ EvaluationResult<SkyValue> result = eval(key);
+ assertThat(result.hasError()).isTrue();
+ assertThat(result.getError(key).getException().getMessage())
+ .contains("'foo/dir' crosses package boundary into package rooted at foo/dir/subpkg");
+ break;
+ default:
+ throw new IllegalStateException(pkgBoundaryMode.toString());
+ }
+ }
+
+ public void testRecursiveTraversalForDirectoryCrossNoFollow() throws Exception {
+ assertRecursiveTraversalForDirectory(SymlinkBehavior.COPY, CROSS);
+ }
+
+ public void testRecursiveTraversalForDirectoryDontCrossNoFollow() throws Exception {
+ assertRecursiveTraversalForDirectory(SymlinkBehavior.COPY, DONT_CROSS);
+ }
+
+ public void testRecursiveTraversalForDirectoryReportErrorNoFollow() throws Exception {
+ assertRecursiveTraversalForDirectory(SymlinkBehavior.COPY, REPORT_ERROR);
+ }
+
+ public void testRecursiveTraversalForDirectoryCrossFollow() throws Exception {
+ assertRecursiveTraversalForDirectory(SymlinkBehavior.DEREFERENCE, CROSS);
+ }
+
+ public void testRecursiveTraversalForDirectoryDontCrossFollow() throws Exception {
+ assertRecursiveTraversalForDirectory(SymlinkBehavior.DEREFERENCE, DONT_CROSS);
+ }
+
+ public void testRecursiveTraversalForDirectoryReportErrorFollow() throws Exception {
+ assertRecursiveTraversalForDirectory(SymlinkBehavior.DEREFERENCE, REPORT_ERROR);
+ }
+
+ private void assertRecursiveTraversalForDirectorySymlink(
+ SymlinkBehavior symlinks, PackageBoundaryMode pkgBoundaryMode) throws Exception {
+ Artifact dir = getSourceArtifact("foo/dir_real");
+ Artifact symlink = getSourceArtifact("foo/dir_sym");
+ createFile(childOf(dir, "file.a"), "blah");
+ RootedPath fileAsym = childOf(dir, "subdir/file.a.sym");
+ createFile(childOf(dir, "subpkg/BUILD"), "blah");
+ createFile(childOf(dir, "subpkg/file.b"), "blah");
+ fileAsym.asPath().getParentDirectory().createDirectory();
+ fileAsym.asPath().createSymbolicLink(new PathFragment("../file.a"));
+ symlink.getPath().createSymbolicLink(new PathFragment("dir_real"));
+
+ FilesetOutputSymlink outA = symlink("output-name/file.a", childOf(symlink, "file.a"));
+ FilesetOutputSymlink outASym = null;
+ FilesetOutputSymlink outBuild =
+ symlink("output-name/subpkg/BUILD", childOf(symlink, "subpkg/BUILD"));
+ FilesetOutputSymlink outB =
+ symlink("output-name/subpkg/file.b", childOf(symlink, "subpkg/file.b"));
+ switch (symlinks) {
+ case COPY:
+ outASym = symlink("output-name/subdir/file.a.sym", "../file.a");
+ break;
+ case DEREFERENCE:
+ outASym = symlink("output-name/subdir/file.a.sym", childOf(dir, "file.a"));
+ break;
+ default:
+ throw new IllegalStateException(symlinks.toString());
+ }
+
+ FilesetTraversalParams params =
+ FilesetTraversalParamsFactory.recursiveTraversalOfDirectory(
+ /*ownerLabel=*/ label("//foo"),
+ /*directoryToTraverse=*/ symlink,
+ /*destPath=*/ new PathFragment("output-name"),
+ /*excludes=*/ null,
+ /*symlinkBehaviorMode=*/ symlinks,
+ /*pkgBoundaryMode=*/ pkgBoundaryMode);
+ switch (pkgBoundaryMode) {
+ case CROSS:
+ assertSymlinksInOrder(params, outA, outASym, outBuild, outB);
+ break;
+ case DONT_CROSS:
+ assertSymlinksInOrder(params, outA, outASym);
+ break;
+ case REPORT_ERROR:
+ SkyKey key = FilesetEntryValue.key(params);
+ EvaluationResult<SkyValue> result = eval(key);
+ assertThat(result.hasError()).isTrue();
+ assertThat(result.getError(key).getException().getMessage())
+ .contains(
+ "'foo/dir_sym' crosses package boundary into package rooted at foo/dir_sym/subpkg");
+ break;
+ default:
+ throw new IllegalStateException(pkgBoundaryMode.toString());
+ }
+ }
+
+ public void testRecursiveTraversalForDirectorySymlinkNoFollowCross() throws Exception {
+ assertRecursiveTraversalForDirectorySymlink(SymlinkBehavior.COPY, CROSS);
+ }
+
+ public void testRecursiveTraversalForDirectorySymlinkNoFollowDontCross() throws Exception {
+ assertRecursiveTraversalForDirectorySymlink(SymlinkBehavior.COPY, DONT_CROSS);
+ }
+
+ public void testRecursiveTraversalForDirectorySymlinkNoFollowReportError() throws Exception {
+ assertRecursiveTraversalForDirectorySymlink(SymlinkBehavior.COPY, REPORT_ERROR);
+ }
+
+ public void testRecursiveTraversalForDirectorySymlinkFollowCross() throws Exception {
+ assertRecursiveTraversalForDirectorySymlink(SymlinkBehavior.DEREFERENCE, CROSS);
+ }
+
+ public void testRecursiveTraversalForDirectorySymlinkFollowDontCross() throws Exception {
+ assertRecursiveTraversalForDirectorySymlink(SymlinkBehavior.DEREFERENCE, DONT_CROSS);
+ }
+
+ public void testRecursiveTraversalForDirectorySymlinkFollowReportError() throws Exception {
+ assertRecursiveTraversalForDirectorySymlink(SymlinkBehavior.DEREFERENCE, REPORT_ERROR);
+ }
+
+ private void assertRecursiveTraversalForPackage(
+ SymlinkBehavior symlinks, PackageBoundaryMode pkgBoundaryMode) throws Exception {
+ Artifact buildFile = createSourceArtifact("foo/BUILD");
+ Artifact subpkgBuildFile = createSourceArtifact("foo/subpkg/BUILD");
+ Artifact subpkgSymlink = getSourceArtifact("foo/subpkg_sym");
+
+ RootedPath fileA = createFile(siblingOf(buildFile, "file.a"), "blah");
+ RootedPath fileAsym = siblingOf(buildFile, "subdir/file.a.sym");
+ RootedPath fileB = createFile(siblingOf(subpkgBuildFile, "file.b"), "blah");
+
+ scratch.dir(fileAsym.asPath().getParentDirectory().getPathString());
+ fileAsym.asPath().createSymbolicLink(new PathFragment("../file.a"));
+ subpkgSymlink.getPath().createSymbolicLink(new PathFragment("subpkg"));
+
+ FilesetOutputSymlink outBuild = symlink("output-name/BUILD", buildFile);
+ FilesetOutputSymlink outA = symlink("output-name/file.a", fileA);
+ FilesetOutputSymlink outAsym = null;
+ FilesetOutputSymlink outSubpkgBuild = symlink("output-name/subpkg/BUILD", subpkgBuildFile);
+ FilesetOutputSymlink outSubpkgB = symlink("output-name/subpkg/file.b", fileB);
+ FilesetOutputSymlink outSubpkgSymBuild;
+ switch (symlinks) {
+ case COPY:
+ outAsym = symlink("output-name/subdir/file.a.sym", "../file.a");
+ outSubpkgSymBuild = symlink("output-name/subpkg_sym", "subpkg");
+ break;
+ case DEREFERENCE:
+ outAsym = symlink("output-name/subdir/file.a.sym", fileA);
+ outSubpkgSymBuild = symlink("output-name/subpkg_sym", getSourceArtifact("foo/subpkg"));
+ break;
+ default:
+ throw new IllegalStateException(symlinks.toString());
+ }
+
+ FilesetTraversalParams params =
+ FilesetTraversalParamsFactory.recursiveTraversalOfPackage(
+ /*ownerLabel=*/ label("//foo"),
+ /*directoryToTraverse=*/ buildFile,
+ /*destPath=*/ new PathFragment("output-name"),
+ /*excludes=*/ null,
+ /*symlinkBehaviorMode=*/ symlinks,
+ /*pkgBoundaryMode=*/ pkgBoundaryMode);
+ switch (pkgBoundaryMode) {
+ case CROSS:
+ assertSymlinksInOrder(
+ params, outBuild, outA, outSubpkgSymBuild, outAsym, outSubpkgBuild, outSubpkgB);
+ break;
+ case DONT_CROSS:
+ assertSymlinksInOrder(params, outBuild, outA, outAsym);
+ break;
+ case REPORT_ERROR:
+ SkyKey key = FilesetEntryValue.key(params);
+ EvaluationResult<SkyValue> result = eval(key);
+ assertThat(result.hasError()).isTrue();
+ assertThat(result.getError(key).getException().getMessage())
+ .contains("'foo' crosses package boundary into package rooted at foo/subpkg");
+ break;
+ default:
+ throw new IllegalStateException(pkgBoundaryMode.toString());
+ }
+ }
+
+ public void testRecursiveTraversalForPackageNoFollowCross() throws Exception {
+ assertRecursiveTraversalForPackage(SymlinkBehavior.COPY, CROSS);
+ }
+
+ public void testRecursiveTraversalForPackageNoFollowDontCross() throws Exception {
+ assertRecursiveTraversalForPackage(SymlinkBehavior.COPY, DONT_CROSS);
+ }
+
+ public void testRecursiveTraversalForPackageNoFollowReportError() throws Exception {
+ assertRecursiveTraversalForPackage(SymlinkBehavior.COPY, REPORT_ERROR);
+ }
+
+ public void testRecursiveTraversalForPackageFollowCross() throws Exception {
+ assertRecursiveTraversalForPackage(SymlinkBehavior.DEREFERENCE, CROSS);
+ }
+
+ public void testRecursiveTraversalForPackageFollowDontCross() throws Exception {
+ assertRecursiveTraversalForPackage(SymlinkBehavior.DEREFERENCE, DONT_CROSS);
+ }
+
+ public void testRecursiveTraversalForPackageFollowReportError() throws Exception {
+ assertRecursiveTraversalForPackage(SymlinkBehavior.DEREFERENCE, REPORT_ERROR);
+ }
+
+ public void testNestedFileFilesetTraversal() throws Exception {
+ Artifact path = getSourceArtifact("foo/bar.file");
+ createFile(path, "blah");
+ FilesetTraversalParams inner =
+ FilesetTraversalParamsFactory.fileTraversal(
+ /*ownerLabel=*/ label("//foo"),
+ /*fileToTraverse=*/ path,
+ /*destPath=*/ new PathFragment("inner-out"),
+ /*symlinkBehaviorMode=*/ SymlinkBehavior.COPY,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+ FilesetTraversalParams outer =
+ FilesetTraversalParamsFactory.nestedTraversal(
+ /*ownerLabel=*/ label("//foo:bar"),
+ /*nested=*/ inner,
+ /*destDir=*/ new PathFragment("outer-out"),
+ /*excludes=*/ null);
+ assertSymlinksInOrder(outer, symlink("outer-out/inner-out", rootedPath(path)));
+ }
+
+ private void assertNestedRecursiveFilesetTraversal(boolean useInnerDir) throws Exception {
+ Artifact dir = getSourceArtifact("foo/dir");
+ RootedPath fileA = createFile(childOf(dir, "file.a"), "hello");
+ RootedPath fileB = createFile(childOf(dir, "file.b"), "hello");
+ RootedPath fileC = createFile(childOf(dir, "sub/file.c"), "world");
+
+ FilesetTraversalParams inner =
+ FilesetTraversalParamsFactory.recursiveTraversalOfDirectory(
+ /*ownerLabel=*/ label("//foo"),
+ /*directoryToTraverse=*/ dir,
+ /*destPath=*/ new PathFragment(useInnerDir ? "inner-dir" : ""),
+ /*excludes=*/ null,
+ /*symlinkBehaviorMode=*/ SymlinkBehavior.COPY,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+ FilesetTraversalParams outer =
+ FilesetTraversalParamsFactory.nestedTraversal(
+ /*ownerLabel=*/ label("//foo"),
+ /*nested=*/ inner,
+ /*destDir=*/ new PathFragment("outer-dir"),
+ ImmutableSet.<String>of("file.a", "sub/file.c"));
+
+ if (useInnerDir) {
+ assertSymlinksInOrder(
+ outer,
+ // no file is excluded, since no files from "inner" are top-level in the outer Fileset
+ symlink("outer-dir/inner-dir/file.a", fileA),
+ symlink("outer-dir/inner-dir/file.b", fileB),
+ symlink("outer-dir/inner-dir/sub/file.c", fileC)); // only top-level files are excluded
+ } else {
+ assertSymlinksInOrder(
+ outer,
+ // file.a can be excluded because it's top-level (there's no output directory for "inner")
+ symlink("outer-dir/file.b", fileB),
+ symlink("outer-dir/sub/file.c", fileC)); // only top-level files could be excluded
+ }
+ }
+
+ public void testNestedRecursiveFilesetTraversalWithInnerDestDir() throws Exception {
+ assertNestedRecursiveFilesetTraversal(true);
+ }
+
+ public void testNestedRecursiveFilesetTraversalWithoutInnerDestDir() throws Exception {
+ assertNestedRecursiveFilesetTraversal(false);
+ }
+
+ public void testFileTraversalForDanglingSymlink() throws Exception {
+ Artifact linkName = getSourceArtifact("foo/dangling.sym");
+ RootedPath linkTarget = createFile(siblingOf(linkName, "target.file"), "blah");
+ linkName.getPath().createSymbolicLink(new PathFragment("target.file"));
+ linkTarget.asPath().delete();
+
+ FilesetTraversalParams params =
+ FilesetTraversalParamsFactory.fileTraversal(
+ /*ownerLabel=*/ label("//foo"),
+ /*fileToTraverse=*/ linkName,
+ /*destPath=*/ new PathFragment("output-name"),
+ /*symlinkBehaviorMode=*/ SymlinkBehavior.COPY,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+ assertSymlinksInOrder(params); // expect empty results
+ }
+
+ public void testFileTraversalForNonExistentFile() throws Exception {
+ Artifact path = getSourceArtifact("foo/non-existent");
+ FilesetTraversalParams params =
+ FilesetTraversalParamsFactory.fileTraversal(
+ /*ownerLabel=*/ label("//foo"),
+ /*fileToTraverse=*/ path,
+ /*destPath=*/ new PathFragment("output-name"),
+ /*symlinkBehaviorMode=*/ SymlinkBehavior.COPY,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+ assertSymlinksInOrder(params); // expect empty results
+ }
+
+ public void testRecursiveTraversalForDanglingSymlink() throws Exception {
+ Artifact linkName = getSourceArtifact("foo/dangling.sym");
+ RootedPath linkTarget = createFile(siblingOf(linkName, "target.file"), "blah");
+ linkName.getPath().createSymbolicLink(new PathFragment("target.file"));
+ linkTarget.asPath().delete();
+
+ FilesetTraversalParams params =
+ FilesetTraversalParamsFactory.recursiveTraversalOfDirectory(
+ /*ownerLabel=*/ label("//foo"),
+ /*directoryToTraverse=*/ linkName,
+ /*destPath=*/ new PathFragment("output-name"),
+ /*excludes=*/ null,
+ /*symlinkBehaviorMode=*/ SymlinkBehavior.COPY,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+ assertSymlinksInOrder(params); // expect empty results
+ }
+
+ public void testRecursiveTraversalForNonExistentFile() throws Exception {
+ Artifact path = getSourceArtifact("foo/non-existent");
+
+ FilesetTraversalParams params =
+ FilesetTraversalParamsFactory.recursiveTraversalOfDirectory(
+ /*ownerLabel=*/ label("//foo"),
+ /*directoryToTraverse=*/ path,
+ /*destPath=*/ new PathFragment("output-name"),
+ /*excludes=*/ null,
+ /*symlinkBehaviorMode=*/ SymlinkBehavior.COPY,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+ assertSymlinksInOrder(params); // expect empty results
+ }
+
+ /**
+ * Tests that the fingerprint is a function of all arguments of the factory method.
+ *
+ * <p>Implementations must provide:
+ * <ul>
+ * <li>two different values (a domain) for each argument of the factory method and whether or not
+ * it is expected to influence the fingerprint
+ * <li>a way to instantiate {@link FilesetTraversalParams} with a given set of arguments from the
+ * specified domains
+ * </ul>
+ *
+ * <p>The tests will instantiate pairs of {@link FilesetTraversalParams} objects with only a given
+ * attribute differing, and observe whether the fingerprints differ (if they are expected to) or
+ * are the same (otherwise).
+ */
+ private abstract static class FingerprintTester {
+ private final Map<String, Domain> domains;
+
+ FingerprintTester(Map<String, Domain> domains) {
+ this.domains = domains;
+ }
+
+ abstract FilesetTraversalParams create(Map<String, ?> kwArgs) throws Exception;
+
+ private Map<String, ?> getDefaultArgs() {
+ return getKwArgs(null);
+ }
+
+ private Map<String, ?> getKwArgs(@Nullable String useAlternateFor) {
+ Map<String, Object> values = new HashMap<>();
+ for (Map.Entry<String, Domain> d : domains.entrySet()) {
+ values.put(
+ d.getKey(),
+ d.getKey().equals(useAlternateFor) ? d.getValue().valueA : d.getValue().valueB);
+ }
+ return values;
+ }
+
+ public void doTest() throws Exception {
+ Fingerprint fp = new Fingerprint();
+
+ create(getDefaultArgs()).fingerprint(fp);
+ String primary = fp.hexDigestAndReset();
+
+ for (String argName : domains.keySet()) {
+ create(getKwArgs(argName)).fingerprint(fp);
+ String secondary = fp.hexDigestAndReset();
+
+ if (domains.get(argName).includedInFingerprint) {
+ assertWithMessage(
+ "Argument '"
+ + argName
+ + "' was expected to be included in the"
+ + " fingerprint, but wasn't")
+ .that(primary)
+ .isNotEqualTo(secondary);
+ } else {
+ assertWithMessage(
+ "Argument '"
+ + argName
+ + "' was expected not to be included in the"
+ + " fingerprint, but was")
+ .that(primary)
+ .isEqualTo(secondary);
+ }
+ }
+ }
+ }
+
+ private static final class Domain {
+ boolean includedInFingerprint;
+ Object valueA;
+ Object valueB;
+
+ Domain(boolean includedInFingerprint, Object valueA, Object valueB) {
+ this.includedInFingerprint = includedInFingerprint;
+ this.valueA = valueA;
+ this.valueB = valueB;
+ }
+ }
+
+ private static Domain partOfFingerprint(Object valueA, Object valueB) {
+ return new Domain(true, valueA, valueB);
+ }
+
+ private static Domain notPartOfFingerprint(Object valueA, Object valueB) {
+ return new Domain(false, valueA, valueB);
+ }
+
+ public void testFingerprintOfFileTraversal() throws Exception {
+ new FingerprintTester(
+ ImmutableMap.<String, Domain>of(
+ "ownerLabel", notPartOfFingerprint("//foo", "//bar"),
+ "fileToTraverse", partOfFingerprint("foo/file.a", "bar/file.b"),
+ "destPath", partOfFingerprint("out1", "out2"),
+ "symlinkBehaviorMode",
+ partOfFingerprint(SymlinkBehavior.COPY, SymlinkBehavior.DEREFERENCE),
+ "pkgBoundaryMode", partOfFingerprint(CROSS, DONT_CROSS))) {
+ @Override
+ FilesetTraversalParams create(Map<String, ?> kwArgs) throws Exception {
+ return FilesetTraversalParamsFactory.fileTraversal(
+ label((String) kwArgs.get("ownerLabel")),
+ getSourceArtifact((String) kwArgs.get("fileToTraverse")),
+ new PathFragment((String) kwArgs.get("destPath")),
+ ((SymlinkBehavior) kwArgs.get("symlinkBehaviorMode")),
+ (PackageBoundaryMode) kwArgs.get("pkgBoundaryMode"));
+ }
+ }.doTest();
+ }
+
+ public void testFingerprintOfDirectoryTraversal() throws Exception {
+ new FingerprintTester(
+ ImmutableMap.<String, Domain>builder()
+ .put("ownerLabel", notPartOfFingerprint("//foo", "//bar"))
+ .put("directoryToTraverse", partOfFingerprint("foo/dir_a", "bar/dir_b"))
+ .put("destPath", partOfFingerprint("out1", "out2"))
+ .put(
+ "excludes",
+ partOfFingerprint(ImmutableSet.<String>of(), ImmutableSet.<String>of("blah")))
+ .put(
+ "symlinkBehaviorMode",
+ partOfFingerprint(SymlinkBehavior.COPY, SymlinkBehavior.DEREFERENCE))
+ .put("pkgBoundaryMode", partOfFingerprint(CROSS, DONT_CROSS))
+ .build()) {
+ @SuppressWarnings("unchecked")
+ @Override
+ FilesetTraversalParams create(Map<String, ?> kwArgs) throws Exception {
+ return FilesetTraversalParamsFactory.recursiveTraversalOfDirectory(
+ label((String) kwArgs.get("ownerLabel")),
+ getSourceArtifact((String) kwArgs.get("directoryToTraverse")),
+ new PathFragment((String) kwArgs.get("destPath")),
+ (Set<String>) kwArgs.get("excludes"),
+ ((SymlinkBehavior) kwArgs.get("symlinkBehaviorMode")),
+ (PackageBoundaryMode) kwArgs.get("pkgBoundaryMode"));
+ }
+ }.doTest();
+ }
+
+ public void testFingerprintOfPackageTraversal() throws Exception {
+ new FingerprintTester(
+ ImmutableMap.<String, Domain>builder()
+ .put("ownerLabel", notPartOfFingerprint("//foo", "//bar"))
+ .put("buildFile", partOfFingerprint("foo/BUILD", "bar/BUILD"))
+ .put("destPath", partOfFingerprint("out1", "out2"))
+ .put(
+ "excludes",
+ partOfFingerprint(ImmutableSet.<String>of(), ImmutableSet.<String>of("blah")))
+ .put(
+ "symlinkBehaviorMode",
+ partOfFingerprint(SymlinkBehavior.COPY, SymlinkBehavior.DEREFERENCE))
+ .put("pkgBoundaryMode", partOfFingerprint(CROSS, DONT_CROSS))
+ .build()) {
+ @SuppressWarnings("unchecked")
+ @Override
+ FilesetTraversalParams create(Map<String, ?> kwArgs) throws Exception {
+ return FilesetTraversalParamsFactory.recursiveTraversalOfPackage(
+ label((String) kwArgs.get("ownerLabel")),
+ getSourceArtifact((String) kwArgs.get("buildFile")),
+ new PathFragment((String) kwArgs.get("destPath")),
+ (Set<String>) kwArgs.get("excludes"),
+ ((SymlinkBehavior) kwArgs.get("symlinkBehaviorMode")),
+ (PackageBoundaryMode) kwArgs.get("pkgBoundaryMode"));
+ }
+ }.doTest();
+ }
+
+ public void testFingerprintOfNestedTraversal() throws Exception {
+ FilesetTraversalParams n1 =
+ FilesetTraversalParamsFactory.fileTraversal(
+ /*ownerLabel=*/ label("//blah"),
+ /*fileToTraverse=*/ getSourceArtifact("blah/file.a"),
+ /*destPath=*/ new PathFragment("output-name"),
+ /*symlinkBehaviorMode=*/ SymlinkBehavior.COPY,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+
+ FilesetTraversalParams n2 =
+ FilesetTraversalParamsFactory.fileTraversal(
+ /*ownerLabel=*/ label("//blah"),
+ /*fileToTraverse=*/ getSourceArtifact("meow/file.b"),
+ /*destPath=*/ new PathFragment("output-name"),
+ /*symlinkBehaviorMode=*/ SymlinkBehavior.COPY,
+ /*pkgBoundaryMode=*/ DONT_CROSS);
+
+ new FingerprintTester(
+ ImmutableMap.<String, Domain>of(
+ "ownerLabel", notPartOfFingerprint("//foo", "//bar"),
+ "nested", partOfFingerprint(n1, n2),
+ "destDir", partOfFingerprint("out1", "out2"),
+ "excludes",
+ partOfFingerprint(ImmutableSet.<String>of(), ImmutableSet.<String>of("x")))) {
+ @SuppressWarnings("unchecked")
+ @Override
+ FilesetTraversalParams create(Map<String, ?> kwArgs) throws Exception {
+ return FilesetTraversalParamsFactory.nestedTraversal(
+ label((String) kwArgs.get("ownerLabel")),
+ (FilesetTraversalParams) kwArgs.get("nested"),
+ new PathFragment((String) kwArgs.get("destDir")),
+ (Set<String>) kwArgs.get("excludes"));
+ }
+ }.doTest();
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FilesystemValueCheckerTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/FilesystemValueCheckerTest.java
new file mode 100644
index 0000000..d917c8e
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/FilesystemValueCheckerTest.java
@@ -0,0 +1,526 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.Runnables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.actions.util.TestAction;
+import com.google.devtools.build.lib.events.NullEventHandler;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.BasicFilesystemDirtinessChecker;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.BatchStat;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+import com.google.devtools.build.skyframe.Differencer.Diff;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.RecordingDifferencer;
+import com.google.devtools.build.skyframe.SequentialBuildDriver;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * Tests for {@link FilesystemValueChecker}.
+ */
+public class FilesystemValueCheckerTest extends TestCase {
+
+ private RecordingDifferencer differencer;
+ private MemoizingEvaluator evaluator;
+ private SequentialBuildDriver driver;
+ private MockFileSystem fs;
+ private Path pkgRoot;
+ private TimestampGranularityMonitor tsgm;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+
+ ImmutableMap.Builder<SkyFunctionName, SkyFunction> skyFunctions = ImmutableMap.builder();
+
+ fs = new MockFileSystem();
+ pkgRoot = fs.getPath("/testroot");
+
+ tsgm = new TimestampGranularityMonitor(BlazeClock.instance());
+ AtomicReference<PathPackageLocator> pkgLocator =
+ new AtomicReference<>(new PathPackageLocator());
+ ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator);
+ skyFunctions.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
+ skyFunctions.put(SkyFunctions.FILE, new FileFunction(pkgLocator, tsgm, externalFilesHelper));
+ skyFunctions.put(
+ SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS, new FileSymlinkCycleUniquenessFunction());
+ skyFunctions.put(
+ SkyFunctions.FILE_SYMLINK_INFINITE_EXPANSION_UNIQUENESS,
+ new FileSymlinkInfiniteExpansionUniquenessFunction());
+ differencer = new RecordingDifferencer();
+ evaluator = new InMemoryMemoizingEvaluator(skyFunctions.build(), differencer);
+ driver = new SequentialBuildDriver(evaluator);
+ PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ public void testEmpty() throws Exception {
+ FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+ assertEmptyDiff(getDirtyFilesystemKeys(checker));
+ }
+
+ public void testSimple() throws Exception {
+ FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+
+ Path path = fs.getPath("/foo");
+ FileSystemUtils.createEmptyFile(path);
+ assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+ SkyKey skyKey =
+ FileStateValue.key(RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("foo")));
+ EvaluationResult<SkyValue> result =
+ driver.evaluate(
+ ImmutableList.of(skyKey),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ assertFalse(result.hasError());
+
+ assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+ FileSystemUtils.writeContentAsLatin1(path, "hello");
+ assertDiffWithNewValues(getDirtyFilesystemKeys(checker), skyKey);
+
+ // The dirty bits are not reset until the FileValues are actually revalidated.
+ assertDiffWithNewValues(getDirtyFilesystemKeys(checker), skyKey);
+
+ differencer.invalidate(ImmutableList.of(skyKey));
+ result =
+ driver.evaluate(
+ ImmutableList.of(skyKey),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ assertFalse(result.hasError());
+ assertEmptyDiff(getDirtyFilesystemKeys(checker));
+ }
+
+ /**
+ * Tests that an already-invalidated value can still be marked changed: symlink points at sym1.
+ * Invalidate symlink by changing sym1 from pointing at path to point to sym2. This only dirties
+ * (rather than changes) symlink because sym2 still points at path, so all symlink stats remain
+ * the same. Then do a null build, change sym1 back to point at path, and change symlink to not be
+ * a symlink anymore. The fact that it is not a symlink should be detected.
+ */
+ public void testDirtySymlink() throws Exception {
+ FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+
+ Path path = fs.getPath("/foo");
+ FileSystemUtils.writeContentAsLatin1(path, "foo contents");
+ // We need the intermediate sym1 and sym2 so that we can dirty a child of symlink without
+ // actually changing the FileValue calculated for symlink (if we changed the contents of foo,
+ // the the FileValue created for symlink would notice, since it stats foo).
+ Path sym1 = fs.getPath("/sym1");
+ Path sym2 = fs.getPath("/sym2");
+ Path symlink = fs.getPath("/bar");
+ FileSystemUtils.ensureSymbolicLink(symlink, sym1);
+ FileSystemUtils.ensureSymbolicLink(sym1, path);
+ FileSystemUtils.ensureSymbolicLink(sym2, path);
+ SkyKey fooKey =
+ FileValue.key(RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("foo")));
+ RootedPath symlinkRootedPath =
+ RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("bar"));
+ SkyKey symlinkKey = FileValue.key(symlinkRootedPath);
+ SkyKey symlinkFileStateKey = FileStateValue.key(symlinkRootedPath);
+ RootedPath sym1RootedPath =
+ RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("sym1"));
+ SkyKey sym1FileStateKey = FileStateValue.key(sym1RootedPath);
+ Iterable<SkyKey> allKeys = ImmutableList.of(symlinkKey, fooKey);
+
+ // First build -- prime the graph.
+ EvaluationResult<FileValue> result =
+ driver.evaluate(
+ allKeys, false, SkyframeExecutor.DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
+ assertFalse(result.hasError());
+ FileValue symlinkValue = result.get(symlinkKey);
+ FileValue fooValue = result.get(fooKey);
+ assertTrue(symlinkValue.toString(), symlinkValue.isSymlink());
+ // Digest is not always available, so use size as a proxy for contents.
+ assertEquals(fooValue.getSize(), symlinkValue.getSize());
+ assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+ // Before second build, move sym1 to point to sym2.
+ assertTrue(sym1.delete());
+ FileSystemUtils.ensureSymbolicLink(sym1, sym2);
+ assertDiffWithNewValues(getDirtyFilesystemKeys(checker), sym1FileStateKey);
+
+ differencer.invalidate(ImmutableList.of(sym1FileStateKey));
+ result =
+ driver.evaluate(
+ ImmutableList.<SkyKey>of(),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ assertFalse(result.hasError());
+ assertDiffWithNewValues(getDirtyFilesystemKeys(checker), sym1FileStateKey);
+
+ // Before third build, move sym1 back to original (so change pruning will prevent signaling of
+ // its parents, but change symlink for real.
+ assertTrue(sym1.delete());
+ FileSystemUtils.ensureSymbolicLink(sym1, path);
+ assertTrue(symlink.delete());
+ FileSystemUtils.writeContentAsLatin1(symlink, "new symlink contents");
+ assertDiffWithNewValues(getDirtyFilesystemKeys(checker), symlinkFileStateKey);
+ differencer.invalidate(ImmutableList.of(symlinkFileStateKey));
+ result =
+ driver.evaluate(
+ allKeys, false, SkyframeExecutor.DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
+ assertFalse(result.hasError());
+ symlinkValue = result.get(symlinkKey);
+ assertFalse(symlinkValue.toString(), symlinkValue.isSymlink());
+ assertEquals(fooValue, result.get(fooKey));
+ assertThat(symlinkValue.getSize()).isNotEqualTo(fooValue.getSize());
+ assertEmptyDiff(getDirtyFilesystemKeys(checker));
+ }
+
+ public void testExplicitFiles() throws Exception {
+ FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+
+ Path path1 = fs.getPath("/foo1");
+ Path path2 = fs.getPath("/foo2");
+ FileSystemUtils.createEmptyFile(path1);
+ FileSystemUtils.createEmptyFile(path2);
+ assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+ SkyKey key1 =
+ FileStateValue.key(
+ RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("foo1")));
+ SkyKey key2 =
+ FileStateValue.key(
+ RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("foo2")));
+ Iterable<SkyKey> skyKeys = ImmutableList.of(key1, key2);
+ EvaluationResult<SkyValue> result =
+ driver.evaluate(
+ skyKeys, false, SkyframeExecutor.DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
+ assertFalse(result.hasError());
+
+ assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+ FileSystemUtils.writeContentAsLatin1(path1, "hello1");
+ FileSystemUtils.writeContentAsLatin1(path1, "hello2");
+ path1.setLastModifiedTime(27);
+ path2.setLastModifiedTime(42);
+ assertDiffWithNewValues(getDirtyFilesystemKeys(checker), key1, key2);
+
+ differencer.invalidate(skyKeys);
+ result =
+ driver.evaluate(
+ skyKeys, false, SkyframeExecutor.DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
+ assertFalse(result.hasError());
+ assertEmptyDiff(getDirtyFilesystemKeys(checker));
+ }
+
+ public void testFileWithIOExceptionNotConsideredDirty() throws Exception {
+ Path path = fs.getPath("/testroot/foo");
+ path.getParentDirectory().createDirectory();
+ path.createSymbolicLink(new PathFragment("bar"));
+
+ fs.readlinkThrowsIoException = true;
+ SkyKey fileKey = FileStateValue.key(RootedPath.toRootedPath(pkgRoot, new PathFragment("foo")));
+ EvaluationResult<SkyValue> result =
+ driver.evaluate(
+ ImmutableList.of(fileKey),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ assertTrue(result.hasError());
+
+ fs.readlinkThrowsIoException = false;
+ FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+ Diff diff = getDirtyFilesystemKeys(checker);
+ assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
+ assertThat(diff.changedKeysWithNewValues()).isEmpty();
+ }
+
+ public void testFilesInCycleNotConsideredDirty() throws Exception {
+ Path path1 = pkgRoot.getRelative("foo1");
+ Path path2 = pkgRoot.getRelative("foo2");
+ Path path3 = pkgRoot.getRelative("foo3");
+ FileSystemUtils.ensureSymbolicLink(path1, path2);
+ FileSystemUtils.ensureSymbolicLink(path2, path3);
+ FileSystemUtils.ensureSymbolicLink(path3, path1);
+ SkyKey fileKey1 = FileValue.key(RootedPath.toRootedPath(pkgRoot, path1));
+
+ EvaluationResult<SkyValue> result =
+ driver.evaluate(
+ ImmutableList.of(fileKey1),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ assertTrue(result.hasError());
+
+ FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+ Diff diff = getDirtyFilesystemKeys(checker);
+ assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
+ assertThat(diff.changedKeysWithNewValues()).isEmpty();
+ }
+
+ public void checkDirtyActions(BatchStat batchStatter, boolean forceDigests) throws Exception {
+ Artifact out1 = createDerivedArtifact("fiz");
+ Artifact out2 = createDerivedArtifact("pop");
+
+ FileSystemUtils.writeContentAsLatin1(out1.getPath(), "hello");
+ FileSystemUtils.writeContentAsLatin1(out2.getPath(), "fizzlepop");
+
+ Action action1 =
+ new TestAction(
+ Runnables.doNothing(), ImmutableSet.<Artifact>of(), ImmutableSet.<Artifact>of(out1));
+ Action action2 =
+ new TestAction(
+ Runnables.doNothing(), ImmutableSet.<Artifact>of(), ImmutableSet.<Artifact>of(out2));
+ differencer.inject(
+ ImmutableMap.<SkyKey, SkyValue>of(
+ ActionExecutionValue.key(action1), actionValue(action1, forceDigests),
+ ActionExecutionValue.key(action2), actionValue(action2, forceDigests)));
+ assertFalse(
+ driver
+ .evaluate(ImmutableList.<SkyKey>of(), false, 1, NullEventHandler.INSTANCE)
+ .hasError());
+ assertThat(new FilesystemValueChecker(evaluator, tsgm, null).getDirtyActionValues(batchStatter))
+ .isEmpty();
+
+ FileSystemUtils.writeContentAsLatin1(out1.getPath(), "goodbye");
+ assertEquals(
+ ActionExecutionValue.key(action1),
+ Iterables.getOnlyElement(
+ new FilesystemValueChecker(evaluator, tsgm, null).getDirtyActionValues(batchStatter)));
+ }
+
+ private Artifact createDerivedArtifact(String relPath) throws IOException {
+ Path outputPath = fs.getPath("/bin");
+ outputPath.createDirectory();
+ return new Artifact(
+ outputPath.getRelative(relPath), Root.asDerivedRoot(fs.getPath("/"), outputPath));
+ }
+
+ public void testDirtyActions() throws Exception {
+ checkDirtyActions(null, false);
+ }
+
+ public void testDirtyActionsBatchStat() throws Exception {
+ checkDirtyActions(
+ new BatchStat() {
+ @Override
+ public List<FileStatusWithDigest> batchStat(
+ boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
+ throws IOException {
+ List<FileStatusWithDigest> stats = new ArrayList<>();
+ for (PathFragment pathFrag : paths) {
+ stats.add(
+ FileStatusWithDigestAdapter.adapt(
+ fs.getRootDirectory().getRelative(pathFrag).statIfFound(Symlinks.NOFOLLOW)));
+ }
+ return stats;
+ }
+ },
+ false);
+ }
+
+ public void testDirtyActionsBatchStatWithDigest() throws Exception {
+ checkDirtyActions(
+ new BatchStat() {
+ @Override
+ public List<FileStatusWithDigest> batchStat(
+ boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
+ throws IOException {
+ List<FileStatusWithDigest> stats = new ArrayList<>();
+ for (PathFragment pathFrag : paths) {
+ final Path path = fs.getRootDirectory().getRelative(pathFrag);
+ stats.add(statWithDigest(path, path.statIfFound(Symlinks.NOFOLLOW)));
+ }
+ return stats;
+ }
+ },
+ true);
+ }
+
+ public void testDirtyActionsBatchStatFallback() throws Exception {
+ checkDirtyActions(
+ new BatchStat() {
+ @Override
+ public List<FileStatusWithDigest> batchStat(
+ boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
+ throws IOException {
+ throw new IOException("try again");
+ }
+ },
+ false);
+ }
+
+ private ActionExecutionValue actionValue(Action action, boolean forceDigest) {
+ Map<Artifact, FileValue> artifactData = new HashMap<>();
+ for (Artifact output : action.getOutputs()) {
+ try {
+ Path path = output.getPath();
+ FileStatusWithDigest stat =
+ forceDigest ? statWithDigest(path, path.statIfFound(Symlinks.NOFOLLOW)) : null;
+ artifactData.put(output, ActionMetadataHandler.fileValueFromArtifact(output, stat, tsgm));
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ return new ActionExecutionValue(artifactData, ImmutableMap.<Artifact, FileArtifactValue>of());
+ }
+
+ public void testPropagatesRuntimeExceptions() throws Exception {
+ Collection<SkyKey> values =
+ ImmutableList.of(FileValue.key(RootedPath.toRootedPath(pkgRoot, new PathFragment("foo"))));
+ driver.evaluate(
+ values, false, SkyframeExecutor.DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
+ FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+
+ assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+ fs.statThrowsRuntimeException = true;
+ try {
+ getDirtyFilesystemKeys(checker);
+ fail();
+ } catch (RuntimeException e) {
+ assertThat(e).hasMessage("bork");
+ }
+ }
+
+ private static void assertEmptyDiff(Diff diff) {
+ assertDiffWithNewValues(diff);
+ }
+
+ private static void assertDiffWithNewValues(Diff diff, SkyKey... keysWithNewValues) {
+ assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
+ assertThat(diff.changedKeysWithNewValues().keySet())
+ .containsExactlyElementsIn(Arrays.asList(keysWithNewValues));
+ }
+
+ private class MockFileSystem extends InMemoryFileSystem {
+
+ boolean statThrowsRuntimeException;
+ boolean readlinkThrowsIoException;
+
+ MockFileSystem() {
+ super();
+ }
+
+ @Override
+ public FileStatus stat(Path path, boolean followSymlinks) throws IOException {
+ if (statThrowsRuntimeException) {
+ throw new RuntimeException("bork");
+ }
+ return super.stat(path, followSymlinks);
+ }
+
+ @Override
+ protected PathFragment readSymbolicLink(Path path) throws IOException {
+ if (readlinkThrowsIoException) {
+ throw new IOException("readlink failed");
+ }
+ return super.readSymbolicLink(path);
+ }
+ }
+
+ private static FileStatusWithDigest statWithDigest(final Path path, final FileStatus stat) {
+ return new FileStatusWithDigest() {
+ @Nullable
+ @Override
+ public byte[] getDigest() throws IOException {
+ return path.getMD5Digest();
+ }
+
+ @Override
+ public boolean isFile() {
+ return stat.isFile();
+ }
+
+ @Override
+ public boolean isSpecialFile() {
+ return stat.isSpecialFile();
+ }
+
+ @Override
+ public boolean isDirectory() {
+ return stat.isDirectory();
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ return stat.isSymbolicLink();
+ }
+
+ @Override
+ public long getSize() throws IOException {
+ return stat.getSize();
+ }
+
+ @Override
+ public long getLastModifiedTime() throws IOException {
+ return stat.getLastModifiedTime();
+ }
+
+ @Override
+ public long getLastChangeTime() throws IOException {
+ return stat.getLastChangeTime();
+ }
+
+ @Override
+ public long getNodeId() throws IOException {
+ return stat.getNodeId();
+ }
+ };
+ }
+
+ private static Diff getDirtyFilesystemKeys(FilesystemValueChecker checker)
+ throws InterruptedException {
+ return checker.getDirtyKeys(new BasicFilesystemDirtinessChecker());
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/GlobFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/GlobFunctionTest.java
new file mode 100644
index 0000000..d70b98a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/GlobFunctionTest.java
@@ -0,0 +1,666 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Functions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.testing.EqualsTester;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.events.NullEventHandler;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.skyframe.GlobValue.InvalidGlobPatternException;
+import com.google.devtools.build.lib.testutil.ManualClock;
+import com.google.devtools.build.lib.testutil.MoreAsserts;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.RecordingDifferencer;
+import com.google.devtools.build.skyframe.SequentialBuildDriver;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * Tests for {@link GlobFunction}.
+ */
+public abstract class GlobFunctionTest extends TestCase {
+ public static class GlobFunctionAlwaysUseDirListingTest extends GlobFunctionTest {
+ @Override
+ protected boolean alwaysUseDirListing() {
+ return true;
+ }
+ }
+
+ public static class RegularGlobFunctionTest extends GlobFunctionTest {
+ @Override
+ protected boolean alwaysUseDirListing() {
+ return false;
+ }
+ }
+
+ private CustomInMemoryFs fs;
+ private MemoizingEvaluator evaluator;
+ private SequentialBuildDriver driver;
+ private RecordingDifferencer differencer;
+ private Path root;
+ private Path pkgPath;
+ private AtomicReference<PathPackageLocator> pkgLocator;
+ private TimestampGranularityMonitor tsgm;
+
+ private static final PackageIdentifier PKG_PATH_ID = PackageIdentifier.createInDefaultRepo("pkg");
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ fs = new CustomInMemoryFs(new ManualClock());
+ root = fs.getRootDirectory().getRelative("root/workspace");
+ pkgPath = root.getRelative(PKG_PATH_ID.getPackageFragment());
+
+ pkgLocator = new AtomicReference<>(new PathPackageLocator(root));
+ tsgm = new TimestampGranularityMonitor(BlazeClock.instance());
+
+ differencer = new RecordingDifferencer();
+ evaluator = new InMemoryMemoizingEvaluator(createFunctionMap(), differencer);
+ driver = new SequentialBuildDriver(evaluator);
+ PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
+ PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
+
+ createTestFiles();
+ }
+
+ private Map<SkyFunctionName, SkyFunction> createFunctionMap() {
+ AtomicReference<ImmutableSet<PackageIdentifier>> deletedPackages =
+ new AtomicReference<>(ImmutableSet.<PackageIdentifier>of());
+ ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator);
+
+ Map<SkyFunctionName, SkyFunction> skyFunctions = new HashMap<>();
+ skyFunctions.put(SkyFunctions.GLOB, new GlobFunction(alwaysUseDirListing()));
+ skyFunctions.put(
+ SkyFunctions.DIRECTORY_LISTING_STATE,
+ new DirectoryListingStateFunction(externalFilesHelper));
+ skyFunctions.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction());
+ skyFunctions.put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(deletedPackages));
+ skyFunctions.put(
+ SkyFunctions.FILE_STATE,
+ new FileStateFunction(
+ new TimestampGranularityMonitor(BlazeClock.instance()), externalFilesHelper));
+ skyFunctions.put(SkyFunctions.FILE, new FileFunction(pkgLocator, tsgm, externalFilesHelper));
+ return skyFunctions;
+ }
+
+ protected abstract boolean alwaysUseDirListing();
+
+ private void createTestFiles() throws IOException {
+ FileSystemUtils.createDirectoryAndParents(pkgPath);
+ FileSystemUtils.createEmptyFile(pkgPath.getRelative("BUILD"));
+ for (String dir :
+ ImmutableList.of(
+ "foo/bar/wiz", "foo/barnacle/wiz", "food/barnacle/wiz", "fool/barnacle/wiz")) {
+ FileSystemUtils.createDirectoryAndParents(pkgPath.getRelative(dir));
+ }
+ FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/bar/wiz/file"));
+
+ // Used for testing the behavior of globbing into nested subpackages.
+ for (String dir : ImmutableList.of("a1/b1/c", "a2/b2/c")) {
+ FileSystemUtils.createDirectoryAndParents(pkgPath.getRelative(dir));
+ }
+ FileSystemUtils.createEmptyFile(pkgPath.getRelative("a2/b2/BUILD"));
+ }
+
+ public void testSimple() throws Exception {
+ assertGlobMatches("food", /* => */ "food");
+ }
+
+ public void testStartsWithStar() throws Exception {
+ assertGlobMatches("*oo", /* => */ "foo");
+ }
+
+ public void testStartsWithStarWithMiddleStar() throws Exception {
+ assertGlobMatches("*f*o", /* => */ "foo");
+ }
+
+ public void testSingleMatchEqual() throws Exception {
+ assertGlobsEqual("*oo", "*f*o"); // both produce "foo"
+ }
+
+ public void testEndsWithStar() throws Exception {
+ assertGlobMatches("foo*", /* => */ "foo", "food", "fool");
+ }
+
+ public void testEndsWithStarWithMiddleStar() throws Exception {
+ assertGlobMatches("f*oo*", /* => */ "foo", "food", "fool");
+ }
+
+ public void testMultipleMatchesEqual() throws Exception {
+ assertGlobsEqual("foo*", "f*oo*"); // both produce "foo", "food", "fool"
+ }
+
+ public void testMiddleStar() throws Exception {
+ assertGlobMatches("f*o", /* => */ "foo");
+ }
+
+ public void testTwoMiddleStars() throws Exception {
+ assertGlobMatches("f*o*o", /* => */ "foo");
+ }
+
+ public void testSingleStarPatternWithNamedChild() throws Exception {
+ assertGlobMatches("*/bar", /* => */ "foo/bar");
+ }
+
+ public void testDeepSubpackages() throws Exception {
+ assertGlobMatches("*/*/c", /* => */ "a1/b1/c");
+ }
+
+ public void testSingleStarPatternWithChildGlob() throws Exception {
+ assertGlobMatches(
+ "*/bar*", /* => */ "foo/bar", "foo/barnacle", "food/barnacle", "fool/barnacle");
+ }
+
+ public void testSingleStarAsChildGlob() throws Exception {
+ assertGlobMatches("foo/*/wiz", /* => */ "foo/bar/wiz", "foo/barnacle/wiz");
+ }
+
+ public void testNoAsteriskAndFilesDontExist() throws Exception {
+ // Note un-UNIX like semantics:
+ assertGlobMatches("ceci/n'est/pas/une/globbe" /* => nothing */);
+ }
+
+ public void testSingleAsteriskUnderNonexistentDirectory() throws Exception {
+ // Note un-UNIX like semantics:
+ assertGlobMatches("not-there/*" /* => nothing */);
+ }
+
+ public void testDifferentGlobsSameResultEqual() throws Exception {
+ // Once the globs are run, it doesn't matter what pattern ran; only the output.
+ assertGlobsEqual("not-there/*", "syzygy/*"); // Both produce nothing.
+ }
+
+ public void testGlobUnderFile() throws Exception {
+ assertGlobMatches("foo/bar/wiz/file/*" /* => nothing */);
+ }
+
+ public void testGlobEqualsHashCode() throws Exception {
+ // Each "equality group" forms a set of elements that are all equals() to one another,
+ // and also produce the same hashCode.
+ new EqualsTester()
+ .addEqualityGroup(runGlob(false, "no-such-file")) // Matches nothing.
+ .addEqualityGroup(runGlob(false, "BUILD"), runGlob(true, "BUILD")) // Matches BUILD.
+ .addEqualityGroup(runGlob(false, "**")) // Matches lots of things.
+ .addEqualityGroup(
+ runGlob(false, "f*o/bar*"),
+ runGlob(false, "foo/bar*")) // Matches foo/bar and foo/barnacle.
+ .testEquals();
+ }
+
+ public void testGlobMissingPackage() throws Exception {
+ // This is a malformed value key, because "missing" is not a package. Nevertheless, we have a
+ // sanity check that building the corresponding GlobValue fails loudly. The test depends on
+ // implementation details of ParallelEvaluator and GlobFunction.
+ SkyKey skyKey =
+ GlobValue.key(
+ PackageIdentifier.createInDefaultRepo("missing"),
+ "foo",
+ false,
+ PathFragment.EMPTY_FRAGMENT);
+ try {
+ driver.evaluate(
+ ImmutableList.of(skyKey),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ fail();
+ } catch (RuntimeException e) {
+ assertThat(e.getMessage())
+ .contains("Unrecoverable error while evaluating node '" + skyKey + "'");
+ Throwable cause = e.getCause();
+ assertThat(cause).isInstanceOf(IllegalStateException.class);
+ assertThat(cause.getMessage()).contains("isn't an existing package");
+ }
+ }
+
+ public void testGlobDoesNotCrossPackageBoundary() throws Exception {
+ FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/BUILD"));
+ // "foo/bar" should not be in the results because foo is a separate package.
+ assertGlobMatches("f*/*", /* => */ "food/barnacle", "fool/barnacle");
+ }
+
+ public void testGlobDirectoryMatchDoesNotCrossPackageBoundary() throws Exception {
+ FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/bar/BUILD"));
+ // "foo/bar" should not be in the results because foo/bar is a separate package.
+ assertGlobMatches("foo/*", /* => */ "foo/barnacle");
+ }
+
+ public void testStarStarDoesNotCrossPackageBoundary() throws Exception {
+ FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/bar/BUILD"));
+ // "foo/bar" should not be in the results because foo/bar is a separate package.
+ assertGlobMatches("foo/**", /* => */ "foo", "foo/barnacle", "foo/barnacle/wiz");
+ }
+
+ private void assertGlobMatches(String pattern, String... expecteds) throws Exception {
+ assertGlobMatches(false, pattern, expecteds);
+ }
+
+ private void assertGlobWithoutDirsMatches(String pattern, String... expecteds) throws Exception {
+ assertGlobMatches(true, pattern, expecteds);
+ }
+
+ private void assertGlobMatches(boolean excludeDirs, String pattern, String... expecteds)
+ throws Exception {
+ MoreAsserts.assertSameContents(
+ ImmutableList.copyOf(expecteds),
+ Iterables.transform(
+ runGlob(excludeDirs, pattern).getMatches(), Functions.toStringFunction()));
+ }
+
+ private void assertGlobsEqual(String pattern1, String pattern2) throws Exception {
+ GlobValue value1 = runGlob(false, pattern1);
+ GlobValue value2 = runGlob(false, pattern2);
+ assertEquals(
+ "GlobValues "
+ + value1.getMatches()
+ + " and "
+ + value2.getMatches()
+ + " should be equal. "
+ + "Patterns: "
+ + pattern1
+ + ","
+ + pattern2,
+ value1,
+ value2);
+ // Just to be paranoid:
+ assertEquals(value1, value1);
+ assertEquals(value2, value2);
+ }
+
+ private GlobValue runGlob(boolean excludeDirs, String pattern) throws Exception {
+ SkyKey skyKey = GlobValue.key(PKG_PATH_ID, pattern, excludeDirs, PathFragment.EMPTY_FRAGMENT);
+ EvaluationResult<SkyValue> result =
+ driver.evaluate(
+ ImmutableList.of(skyKey),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ if (result.hasError()) {
+ throw result.getError().getException();
+ }
+ return (GlobValue) result.get(skyKey);
+ }
+
+ public void testGlobWithoutWildcards() throws Exception {
+ String pattern = "foo/bar/wiz/file";
+
+ assertGlobMatches(pattern, "foo/bar/wiz/file");
+ // Ensure that the glob depends on the FileValue and not on the DirectoryListingValue.
+ pkgPath.getRelative("foo/bar/wiz/file").delete();
+ // Nothing has been invalidated yet, so the cached result is returned.
+ assertGlobMatches(pattern, "foo/bar/wiz/file");
+
+ if (alwaysUseDirListing()) {
+ differencer.invalidate(
+ ImmutableList.of(
+ FileStateValue.key(
+ RootedPath.toRootedPath(root, pkgPath.getRelative("foo/bar/wiz/file")))));
+ // The result should not rely on the FileStateValue, so it's still a cache hit.
+ assertGlobMatches(pattern, "foo/bar/wiz/file");
+
+ differencer.invalidate(
+ ImmutableList.of(
+ DirectoryListingStateValue.key(
+ RootedPath.toRootedPath(root, pkgPath.getRelative("foo/bar/wiz")))));
+ // This should have invalidated the glob result.
+ assertGlobMatches(pattern /* => nothing */);
+ } else {
+ differencer.invalidate(
+ ImmutableList.of(
+ DirectoryListingStateValue.key(
+ RootedPath.toRootedPath(root, pkgPath.getRelative("foo/bar/wiz")))));
+ // The result should not rely on the DirectoryListingValue, so it's still a cache hit.
+ assertGlobMatches(pattern, "foo/bar/wiz/file");
+
+ differencer.invalidate(
+ ImmutableList.of(
+ FileStateValue.key(
+ RootedPath.toRootedPath(root, pkgPath.getRelative("foo/bar/wiz/file")))));
+ // This should have invalidated the glob result.
+ assertGlobMatches(pattern /* => nothing */);
+ }
+ }
+
+ public void testIllegalPatterns() throws Exception {
+ assertIllegalPattern("(illegal) pattern");
+ assertIllegalPattern("[illegal pattern");
+ assertIllegalPattern("}illegal pattern");
+ assertIllegalPattern("foo**bar");
+ assertIllegalPattern("?");
+ assertIllegalPattern("");
+ assertIllegalPattern(".");
+ assertIllegalPattern("/foo");
+ assertIllegalPattern("./foo");
+ assertIllegalPattern("foo/");
+ assertIllegalPattern("foo/./bar");
+ assertIllegalPattern("../foo/bar");
+ assertIllegalPattern("foo//bar");
+ }
+
+ public void testIllegalRecursivePatterns() throws Exception {
+ for (String prefix : Lists.newArrayList("", "*/", "**/", "ba/")) {
+ String suffix = ("/" + prefix).substring(0, prefix.length());
+ for (String pattern : Lists.newArrayList("**fo", "fo**", "**fo**", "fo**fo", "fo**fo**fo")) {
+ assertIllegalPattern(prefix + pattern);
+ assertIllegalPattern(pattern + suffix);
+ }
+ }
+ }
+
+ private void assertIllegalPattern(String pattern) {
+ try {
+ GlobValue.key(PKG_PATH_ID, pattern, false, PathFragment.EMPTY_FRAGMENT);
+ fail("invalid pattern not detected: " + pattern);
+ } catch (InvalidGlobPatternException e) {
+ // Expected.
+ }
+ }
+
+ /**
+ * Tests that globs can contain Java regular expression special characters
+ */
+ public void testSpecialRegexCharacter() throws Exception {
+ Path aDotB = pkgPath.getChild("a.b");
+ FileSystemUtils.createEmptyFile(aDotB);
+ FileSystemUtils.createEmptyFile(pkgPath.getChild("aab"));
+ // Note: this contains two asterisks because otherwise a RE is not built,
+ // as an optimization.
+ assertThat(UnixGlob.forPath(pkgPath).addPattern("*a.b*").globInterruptible())
+ .containsExactly(aDotB);
+ }
+
+ public void testMatchesCallWithNoCache() {
+ assertTrue(UnixGlob.matches("*a*b", "CaCb", null));
+ }
+
+ public void testHiddenFiles() throws Exception {
+ for (String dir : ImmutableList.of(".hidden", "..also.hidden", "not.hidden")) {
+ FileSystemUtils.createDirectoryAndParents(pkgPath.getRelative(dir));
+ }
+ // Note that these are not in the result: ".", ".."
+ assertGlobMatches(
+ "*", "a1", "a2", "not.hidden", "foo", "fool", "food", "BUILD", ".hidden", "..also.hidden");
+ assertGlobMatches("*.hidden", "not.hidden");
+ }
+
+ public void testDoubleStar() throws Exception {
+ assertGlobMatches(
+ "**",
+ "",
+ "BUILD",
+ "a1",
+ "a1/b1",
+ "a1/b1/c",
+ "a2",
+ "foo",
+ "foo/bar",
+ "foo/bar/wiz",
+ "foo/bar/wiz/file",
+ "foo/barnacle",
+ "foo/barnacle/wiz",
+ "food",
+ "food/barnacle",
+ "food/barnacle/wiz",
+ "fool",
+ "fool/barnacle",
+ "fool/barnacle/wiz");
+ }
+
+ public void testDoubleStarExcludeDirs() throws Exception {
+ assertGlobWithoutDirsMatches("**", "BUILD", "foo/bar/wiz/file");
+ }
+
+ public void testDoubleDoubleStar() throws Exception {
+ assertGlobMatches(
+ "**/**",
+ "",
+ "BUILD",
+ "a1",
+ "a1/b1",
+ "a1/b1/c",
+ "a2",
+ "foo",
+ "foo/bar",
+ "foo/bar/wiz",
+ "foo/bar/wiz/file",
+ "foo/barnacle",
+ "foo/barnacle/wiz",
+ "food",
+ "food/barnacle",
+ "food/barnacle/wiz",
+ "fool",
+ "fool/barnacle",
+ "fool/barnacle/wiz");
+ }
+
+ public void testDirectoryWithDoubleStar() throws Exception {
+ assertGlobMatches(
+ "foo/**",
+ "foo",
+ "foo/bar",
+ "foo/bar/wiz",
+ "foo/bar/wiz/file",
+ "foo/barnacle",
+ "foo/barnacle/wiz");
+ }
+
+ public void testDoubleStarPatternWithNamedChild() throws Exception {
+ assertGlobMatches("**/bar", "foo/bar");
+ }
+
+ public void testDoubleStarPatternWithChildGlob() throws Exception {
+ assertGlobMatches("**/ba*", "foo/bar", "foo/barnacle", "food/barnacle", "fool/barnacle");
+ }
+
+ public void testDoubleStarAsChildGlob() throws Exception {
+ FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/barnacle/wiz/wiz"));
+ FileSystemUtils.createDirectoryAndParents(pkgPath.getRelative("foo/barnacle/baz/wiz"));
+
+ assertGlobMatches(
+ "foo/**/wiz",
+ "foo/bar/wiz",
+ "foo/barnacle/baz/wiz",
+ "foo/barnacle/wiz",
+ "foo/barnacle/wiz/wiz");
+ }
+
+ public void testDoubleStarUnderNonexistentDirectory() throws Exception {
+ assertGlobMatches("not-there/**" /* => nothing */);
+ }
+
+ public void testDoubleStarUnderFile() throws Exception {
+ assertGlobMatches("foo/bar/wiz/file/**" /* => nothing */);
+ }
+
+ /** Regression test for b/13319874: Directory listing crash. */
+ public void testResilienceToFilesystemInconsistencies_DirectoryExistence() throws Exception {
+ long nodeId = pkgPath.getRelative("BUILD").stat().getNodeId();
+ // Our custom filesystem says "pkgPath/BUILD" exists but "pkgPath" does not exist.
+ fs.stubStat(pkgPath, null);
+ RootedPath pkgRootedPath = RootedPath.toRootedPath(root, pkgPath);
+ FileStateValue pkgDirFileStateValue = FileStateValue.create(pkgRootedPath, tsgm);
+ FileValue pkgDirValue =
+ FileValue.value(pkgRootedPath, pkgDirFileStateValue, pkgRootedPath, pkgDirFileStateValue);
+ differencer.inject(ImmutableMap.of(FileValue.key(pkgRootedPath), pkgDirValue));
+ String expectedMessage =
+ "Some filesystem operations implied /root/workspace/pkg/BUILD was a "
+ + "regular file with size of 0 and mtime of 0 and nodeId of "
+ + nodeId
+ + " and mtime of 0 "
+ + "but others made us think it was a nonexistent path";
+ SkyKey skyKey = GlobValue.key(PKG_PATH_ID, "*/foo", false, PathFragment.EMPTY_FRAGMENT);
+ EvaluationResult<GlobValue> result =
+ driver.evaluate(
+ ImmutableList.of(skyKey),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ assertTrue(result.hasError());
+ ErrorInfo errorInfo = result.getError(skyKey);
+ assertThat(errorInfo.getException()).isInstanceOf(InconsistentFilesystemException.class);
+ assertThat(errorInfo.getException().getMessage()).contains(expectedMessage);
+ }
+
+ public void testResilienceToFilesystemInconsistencies_SubdirectoryExistence() throws Exception {
+ // Our custom filesystem says directory "pkgPath/foo/bar" contains a subdirectory "wiz" but a
+ // direct stat on "pkgPath/foo/bar/wiz" says it does not exist.
+ Path fooBarDir = pkgPath.getRelative("foo/bar");
+ fs.stubStat(fooBarDir.getRelative("wiz"), null);
+ RootedPath fooBarDirRootedPath = RootedPath.toRootedPath(root, fooBarDir);
+ SkyValue fooBarDirListingValue =
+ DirectoryListingStateValue.createForTesting(
+ ImmutableList.of(new Dirent("wiz", Dirent.Type.DIRECTORY)));
+ differencer.inject(
+ ImmutableMap.of(
+ DirectoryListingStateValue.key(fooBarDirRootedPath), fooBarDirListingValue));
+ String expectedMessage = "/root/workspace/pkg/foo/bar/wiz is no longer an existing directory.";
+ SkyKey skyKey = GlobValue.key(PKG_PATH_ID, "**/wiz", false, PathFragment.EMPTY_FRAGMENT);
+ EvaluationResult<GlobValue> result =
+ driver.evaluate(
+ ImmutableList.of(skyKey),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ assertTrue(result.hasError());
+ ErrorInfo errorInfo = result.getError(skyKey);
+ assertThat(errorInfo.getException()).isInstanceOf(InconsistentFilesystemException.class);
+ assertThat(errorInfo.getException().getMessage()).contains(expectedMessage);
+ }
+
+ public void testResilienceToFilesystemInconsistencies_SymlinkType() throws Exception {
+ RootedPath wizRootedPath = RootedPath.toRootedPath(root, pkgPath.getRelative("foo/bar/wiz"));
+ RootedPath fileRootedPath =
+ RootedPath.toRootedPath(root, pkgPath.getRelative("foo/bar/wiz/file"));
+ final FileStatus realStat = fileRootedPath.asPath().stat();
+ fs.stubStat(
+ fileRootedPath.asPath(),
+ new FileStatus() {
+
+ @Override
+ public boolean isFile() {
+ // The stat says foo/bar/wiz/file is a real file, not a symlink.
+ return true;
+ }
+
+ @Override
+ public boolean isSpecialFile() {
+ return false;
+ }
+
+ @Override
+ public boolean isDirectory() {
+ return false;
+ }
+
+ @Override
+ public boolean isSymbolicLink() {
+ return false;
+ }
+
+ @Override
+ public long getSize() throws IOException {
+ return realStat.getSize();
+ }
+
+ @Override
+ public long getLastModifiedTime() throws IOException {
+ return realStat.getLastModifiedTime();
+ }
+
+ @Override
+ public long getLastChangeTime() throws IOException {
+ return realStat.getLastChangeTime();
+ }
+
+ @Override
+ public long getNodeId() throws IOException {
+ return realStat.getNodeId();
+ }
+ });
+ // But the dir listing say foo/bar/wiz/file is a symlink.
+ SkyValue wizDirListingValue =
+ DirectoryListingStateValue.createForTesting(
+ ImmutableList.of(new Dirent("file", Dirent.Type.SYMLINK)));
+ differencer.inject(
+ ImmutableMap.of(DirectoryListingStateValue.key(wizRootedPath), wizDirListingValue));
+ String expectedMessage =
+ "readdir and stat disagree about whether " + fileRootedPath.asPath() + " is a symlink";
+ SkyKey skyKey = GlobValue.key(PKG_PATH_ID, "foo/bar/wiz/*", false, PathFragment.EMPTY_FRAGMENT);
+ EvaluationResult<GlobValue> result =
+ driver.evaluate(
+ ImmutableList.of(skyKey),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ assertTrue(result.hasError());
+ ErrorInfo errorInfo = result.getError(skyKey);
+ assertThat(errorInfo.getException()).isInstanceOf(InconsistentFilesystemException.class);
+ assertThat(errorInfo.getException().getMessage()).contains(expectedMessage);
+ }
+
+ private class CustomInMemoryFs extends InMemoryFileSystem {
+
+ private Map<Path, FileStatus> stubbedStats = Maps.newHashMap();
+
+ public CustomInMemoryFs(ManualClock manualClock) {
+ super(manualClock);
+ }
+
+ public void stubStat(Path path, @Nullable FileStatus stubbedResult) {
+ stubbedStats.put(path, stubbedResult);
+ }
+
+ @Override
+ public FileStatus stat(Path path, boolean followSymlinks) throws IOException {
+ if (stubbedStats.containsKey(path)) {
+ return stubbedStats.get(path);
+ }
+ return super.stat(path, followSymlinks);
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionSmartNegationTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionSmartNegationTest.java
new file mode 100644
index 0000000..bfa2d8a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionSmartNegationTest.java
@@ -0,0 +1,122 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.WalkableGraph;
+
+import java.io.IOException;
+
+/** Tests for {@link PrepareDepsOfPatternsFunction}. */
+public class PrepareDepsOfPatternsFunctionSmartNegationTest extends BuildViewTestCase {
+
+ private static SkyKey getKeyForLabel(Label label) {
+ // Note that these tests used to look for TargetMarker SkyKeys before TargetMarker was
+ // inlined in TransitiveTraversalFunction. Because TargetMarker is now inlined, it doesn't
+ // appear in the graph. Instead, these tests now look for TransitiveTraversal keys.
+ return TransitiveTraversalValue.key(label);
+ }
+
+ public void testRecursiveEvaluationFailsOnBadBuildFile() throws Exception {
+ // Given a well-formed package "//foo" and a malformed package "//foo/foo",
+ createFooAndFooFoo();
+
+ // Given a target pattern sequence consisting of a recursive pattern for "//foo/...",
+ ImmutableList<String> patternSequence = ImmutableList.of("//foo/...");
+
+ // When PrepareDepsOfPatternsFunction completes evaluation (with no error because it was
+ // recovered from),
+ WalkableGraph walkableGraph =
+ getGraphFromPatternsEvaluation(
+ patternSequence, /*successExpected=*/ true, /*keepGoing=*/ true);
+
+ // Then the graph contains package values for "//foo" and "//foo/foo",
+ assertTrue(walkableGraph.exists(PackageValue.key(PackageIdentifier.parse("foo"))));
+ assertTrue(walkableGraph.exists(PackageValue.key(PackageIdentifier.parse("foo/foo"))));
+
+ // But the graph does not contain a value for the target "//foo/foo:foofoo".
+ assertFalse(walkableGraph.exists(getKeyForLabel(Label.create("foo/foo", "foofoo"))));
+ }
+
+ public void testNegativePatternBlocksPatternEvaluation() throws Exception {
+ // Given a well-formed package "//foo" and a malformed package "//foo/foo",
+ createFooAndFooFoo();
+
+ // Given a target pattern sequence consisting of a recursive pattern for "//foo/..." followed
+ // by a negative pattern for the malformed package,
+ ImmutableList<String> patternSequence = ImmutableList.of("//foo/...", "-//foo/foo/...");
+
+ // When PrepareDepsOfPatternsFunction completes evaluation (successfully),
+ WalkableGraph walkableGraph =
+ getGraphFromPatternsEvaluation(
+ patternSequence, /*successExpected=*/ true, /*keepGoing=*/ true);
+
+ // Then the graph contains a package value for "//foo",
+ assertTrue(walkableGraph.exists(PackageValue.key(PackageIdentifier.parse("foo"))));
+
+ // But no package value for "//foo/foo",
+ assertFalse(walkableGraph.exists(PackageValue.key(PackageIdentifier.parse("foo/foo"))));
+
+ // And the graph does not contain a value for the target "//foo/foo:foofoo".
+ Label label = Label.create("foo/foo", "foofoo");
+ assertFalse(walkableGraph.exists(getKeyForLabel(label)));
+ }
+
+ public void testNegativeNonTBDPatternsAreSkippedWithWarnings() throws Exception {
+ // Given a target pattern sequence with a negative non-TBD pattern,
+ ImmutableList<String> patternSequence = ImmutableList.of("-//foo/bar");
+
+ // When PrepareDepsOfPatternsFunction completes evaluation,
+ getGraphFromPatternsEvaluation(patternSequence, /*successExpected=*/ true, /*keepGoing=*/ true);
+
+ // Then a event is published that says that negative non-TBD patterns are skipped.
+ assertContainsEvent(
+ "Skipping '-//foo/bar': Negative target patterns of types other than \"targets below "
+ + "directory\" are not permitted.");
+ }
+
+ // Helpers:
+
+ private WalkableGraph getGraphFromPatternsEvaluation(
+ ImmutableList<String> patternSequence, boolean successExpected, boolean keepGoing)
+ throws InterruptedException {
+ SkyKey independentTarget = PrepareDepsOfPatternsValue.key(patternSequence, "");
+ ImmutableList<SkyKey> singletonTargetPattern = ImmutableList.of(independentTarget);
+
+ // When PrepareDepsOfPatternsFunction completes evaluation,
+ EvaluationResult<SkyValue> evaluationResult =
+ getSkyframeExecutor()
+ .getDriverForTesting()
+ .evaluate(singletonTargetPattern, keepGoing, LOADING_PHASE_THREADS, eventCollector);
+ // The evaluation has no errors if success was expected.
+ assertThat(evaluationResult.hasError()).isNotEqualTo(successExpected);
+ return Preconditions.checkNotNull(evaluationResult.getWalkableGraph());
+ }
+
+ private void createFooAndFooFoo() throws IOException {
+ scratch.file(
+ "foo/BUILD", "genrule(name = 'foo',", " outs = ['out.txt'],", " cmd = 'touch $@')");
+ scratch.file(
+ "foo/foo/BUILD", "genrule(name = 'foofoo',", " This isn't even remotely grammatical.)");
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java
new file mode 100644
index 0000000..d7b9e5f
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java
@@ -0,0 +1,273 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.WalkableGraph;
+
+import java.io.IOException;
+
+/** Tests for {@link com.google.devtools.build.lib.skyframe.PrepareDepsOfPatternsFunction}. */
+public class PrepareDepsOfPatternsFunctionTest extends BuildViewTestCase {
+
+ private static SkyKey getKeyForLabel(Label label) {
+ // Note that these tests used to look for TargetMarker SkyKeys before TargetMarker was
+ // inlined in TransitiveTraversalFunction. Because TargetMarker is now inlined, it doesn't
+ // appear in the graph. Instead, these tests now look for TransitiveTraversal keys.
+ return TransitiveTraversalValue.key(label);
+ }
+
+ public void testFunctionLoadsTargetAndNotUnspecifiedTargets() throws Exception {
+ // Given a package "//foo" with independent target rules ":foo" and ":foo2",
+ createFooAndFoo2(/*dependent=*/ false);
+
+ // Given a target pattern sequence consisting of a single-target pattern for "//foo",
+ ImmutableList<String> patternSequence = ImmutableList.of("//foo");
+
+ // When PrepareDepsOfPatternsFunction successfully completes evaluation,
+ WalkableGraph walkableGraph =
+ getGraphFromPatternsEvaluation(patternSequence, /*successExpected=*/ true);
+
+ // Then the graph contains a value for the target "//foo:foo",
+ assertValidValue(walkableGraph, getKeyForLabel(Label.create("foo", "foo")));
+
+ // And the graph does not contain a value for the target "//foo:foo2".
+ assertFalse(walkableGraph.exists(getKeyForLabel(Label.create("foo", "foo2"))));
+ }
+
+ public void testFunctionLoadsTargetDependencies() throws Exception {
+ // Given a package "//foo" with target rules ":foo" and ":foo2",
+ // And given ":foo" depends on ":foo2",
+ createFooAndFoo2(/*dependent=*/ true);
+
+ // Given a target pattern sequence consisting of a single-target pattern for "//foo",
+ ImmutableList<String> patternSequence = ImmutableList.of("//foo");
+
+ // When PrepareDepsOfPatternsFunction successfully completes evaluation,
+ WalkableGraph walkableGraph =
+ getGraphFromPatternsEvaluation(patternSequence, /*successExpected=*/ true);
+
+ // Then the graph contains an entry for ":foo"'s dependency, ":foo2".
+ assertValidValue(walkableGraph, getKeyForLabel(Label.create("foo", "foo2")));
+ }
+
+ public void testFunctionExpandsTargetPatterns() throws Exception {
+ // Given a package "//foo" with independent target rules ":foo" and ":foo2",
+ createFooAndFoo2(/*dependent=*/ false);
+
+ // Given a target pattern sequence consisting of a pattern for "//foo:*",
+ ImmutableList<String> patternSequence = ImmutableList.of("//foo:*");
+
+ // When PrepareDepsOfPatternsFunction successfully completes evaluation,
+ WalkableGraph walkableGraph =
+ getGraphFromPatternsEvaluation(patternSequence, /*successExpected=*/ true);
+
+ // Then the graph contains an entry for ":foo" and ":foo2".
+ assertValidValue(walkableGraph, getKeyForLabel(Label.create("foo", "foo")));
+ assertValidValue(walkableGraph, getKeyForLabel(Label.create("foo", "foo2")));
+ }
+
+ public void testTargetParsingException() throws Exception {
+ // Given no packages, and a target pattern sequence referring to a non-existent target,
+ String nonexistentTarget = "//foo:foo";
+ ImmutableList<String> patternSequence = ImmutableList.of(nonexistentTarget);
+
+ // When PrepareDepsOfPatternsFunction completes evaluation,
+ WalkableGraph walkableGraph =
+ getGraphFromPatternsEvaluation(patternSequence, /*successExpected=*/ false);
+
+ // Then the graph does not contain an entry for ":foo",
+ assertFalse(walkableGraph.exists(getKeyForLabel(Label.create("foo", "foo"))));
+ }
+
+ public void testDependencyTraversalNoSuchPackageException() throws Exception {
+ // Given a package "//foo" with a target ":foo" that has a dependency on a non-existent target
+ // "//bar:bar" in a non-existent package "//bar",
+ createFooWithDependencyOnMissingBarPackage();
+
+ // Given a target pattern sequence consisting of a single-target pattern for "//foo",
+ ImmutableList<String> patternSequence = ImmutableList.of("//foo");
+
+ // When PrepareDepsOfPatternsFunction completes evaluation,
+ WalkableGraph walkableGraph =
+ getGraphFromPatternsEvaluation(patternSequence, /*successExpected=*/ false);
+
+ // Then the graph contains an entry for ":foo",
+ assertValidValue(
+ walkableGraph,
+ getKeyForLabel(Label.create("foo", "foo")),
+ /*expectTransitiveException=*/ true);
+
+ // And an entry with a NoSuchPackageException for "//bar:bar",
+ Exception e = assertException(walkableGraph, getKeyForLabel(Label.create("bar", "bar")));
+ assertThat(e).isInstanceOf(NoSuchPackageException.class);
+ }
+
+ public void testDependencyTraversalNoSuchTargetException() throws Exception {
+ // Given a package "//foo" with a target ":foo" that has a dependency on a non-existent target
+ // "//bar:bar" in an existing package "//bar",
+ createFooWithDependencyOnBarPackageWithMissingTarget();
+
+ // Given a target pattern sequence consisting of a single-target pattern for "//foo",
+ ImmutableList<String> patternSequence = ImmutableList.of("//foo");
+
+ // When PrepareDepsOfPatternsFunction completes evaluation,
+ WalkableGraph walkableGraph =
+ getGraphFromPatternsEvaluation(patternSequence, /*successExpected=*/ false);
+
+ // Then the graph contains an entry for ":foo" which has both a value and an exception,
+ assertValidValue(
+ walkableGraph,
+ getKeyForLabel(Label.create("foo", "foo")),
+ /*expectTransitiveException=*/ true);
+
+ // And an entry with a NoSuchTargetException for "//bar:bar",
+ Exception e = assertException(walkableGraph, getKeyForLabel(Label.create("bar", "bar")));
+ assertThat(e).isInstanceOf(NoSuchTargetException.class);
+ }
+
+ public void testParsingProblemsKeepGoing() throws Exception {
+ parsingProblem(/*keepGoing=*/ true);
+ }
+
+ /**
+ * PrepareDepsOfPatternsFunction always keeps going despite any target pattern parsing errors,
+ * in keeping with the original behavior of {@link SkyframeExecutor#prepareAndGet}, which
+ * always used {@code keepGoing=true} during target pattern parsing because it was responsible
+ * for ensuring that queries had a complete graph to work on.
+ */
+ public void testParsingProblemsNoKeepGoing() throws Exception {
+ parsingProblem(/*keepGoing=*/ false);
+ }
+
+ private void parsingProblem(boolean keepGoing) throws Exception {
+ // Given a package "//foo" with target rule ":foo",
+ createFooAndFoo2(/*dependent=*/ false);
+
+ // Given a target pattern sequence consisting of a pattern with parsing problems followed by
+ // a legit target pattern,
+ String bogusPattern = "//foo/....";
+ ImmutableList<String> patternSequence = ImmutableList.of(bogusPattern, "//foo:foo");
+
+ // When PrepareDepsOfPatternsFunction runs in the selected keep-going mode,
+ WalkableGraph walkableGraph =
+ getGraphFromPatternsEvaluation(
+ patternSequence, /*successExpected=*/ true, /*keepGoing=*/ keepGoing);
+
+ // Then it skips evaluation of the malformed target pattern, but logs about it,
+ assertContainsEvent("Skipping '" + bogusPattern + "': ");
+
+ // And then the graph contains a value for the legit target pattern's target "//foo:foo".
+ assertTrue(walkableGraph.exists(getKeyForLabel(Label.create("foo", "foo"))));
+ }
+
+ // Helpers:
+
+ private WalkableGraph getGraphFromPatternsEvaluation(
+ ImmutableList<String> patternSequence, boolean successExpected) throws InterruptedException {
+ return getGraphFromPatternsEvaluation(patternSequence, successExpected, /*keepGoing=*/ true);
+ }
+
+ private WalkableGraph getGraphFromPatternsEvaluation(
+ ImmutableList<String> patternSequence, boolean successExpected, boolean keepGoing)
+ throws InterruptedException {
+ SkyKey independentTarget = PrepareDepsOfPatternsValue.key(patternSequence, "");
+ ImmutableList<SkyKey> singletonTargetPattern = ImmutableList.of(independentTarget);
+
+ // When PrepareDepsOfPatternsFunction completes evaluation,
+ EvaluationResult<SkyValue> evaluationResult =
+ getSkyframeExecutor()
+ .getDriverForTesting()
+ .evaluate(singletonTargetPattern, keepGoing, LOADING_PHASE_THREADS, eventCollector);
+
+ if (successExpected) {
+ // Then the evaluation completed successfully.
+ assertFalse(evaluationResult.hasError());
+ } else {
+ // Then the evaluation resulted in some errors.
+ assertTrue(evaluationResult.hasError());
+ }
+
+ return Preconditions.checkNotNull(evaluationResult.getWalkableGraph());
+ }
+
+ private void createFooAndFoo2(boolean dependent) throws IOException {
+ String dependencyIfAny = dependent ? "srcs = [':foo2']," : "";
+ scratch.file(
+ "foo/BUILD",
+ "genrule(name = 'foo',",
+ dependencyIfAny,
+ " outs = ['out.txt'],",
+ " cmd = 'touch $@')",
+ "genrule(name = 'foo2',",
+ " outs = ['out2.txt'],",
+ " cmd = 'touch $@')");
+ }
+
+ private void createFooWithDependencyOnMissingBarPackage() throws IOException {
+ scratch.file(
+ "foo/BUILD",
+ "genrule(name = 'foo',",
+ " srcs = ['//bar:bar'],",
+ " outs = ['out.txt'],",
+ " cmd = 'touch $@')");
+ }
+
+ private void createFooWithDependencyOnBarPackageWithMissingTarget() throws IOException {
+ scratch.file(
+ "foo/BUILD",
+ "genrule(name = 'foo',",
+ " srcs = ['//bar:bar'],",
+ " outs = ['out.txt'],",
+ " cmd = 'touch $@')");
+ scratch.file("bar/BUILD");
+ }
+
+ private void assertValidValue(WalkableGraph graph, SkyKey key) {
+ assertValidValue(graph, key, /*expectTransitiveException=*/ false);
+ }
+
+ /**
+ * A node in the walkable graph may have both a value and an exception. This happens when one
+ * of a node's transitive dependencies throws an exception, but its parent recovers from it.
+ */
+ private void assertValidValue(
+ WalkableGraph graph, SkyKey key, boolean expectTransitiveException) {
+ assertTrue(graph.exists(key));
+ assertNotNull(graph.getValue(key));
+ if (expectTransitiveException) {
+ assertNotNull(graph.getException(key));
+ } else {
+ assertNull(graph.getException(key));
+ }
+ }
+
+ private Exception assertException(WalkableGraph graph, SkyKey key) {
+ assertTrue(graph.exists(key));
+ assertNull(graph.getValue(key));
+ Exception exception = graph.getException(key);
+ assertNotNull(exception);
+ return exception;
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunctionTest.java
new file mode 100644
index 0000000..fb46d73
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/RecursiveFilesystemTraversalFunctionTest.java
@@ -0,0 +1,757 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.lib.actions.FilesetTraversalParams.PackageBoundaryMode.CROSS;
+import static com.google.devtools.build.lib.actions.FilesetTraversalParams.PackageBoundaryMode.DONT_CROSS;
+import static com.google.devtools.build.lib.actions.FilesetTraversalParams.PackageBoundaryMode.REPORT_ERROR;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.FilesetTraversalParams.PackageBoundaryMode;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.events.NullEventHandler;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile;
+import com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.TraversalRequest;
+import com.google.devtools.build.lib.testutil.FoundationTestCase;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.RecordingDifferencer;
+import com.google.devtools.build.skyframe.SequentialBuildDriver;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+
+/** Tests for {@link RecursiveFilesystemTraversalFunction}. */
+public final class RecursiveFilesystemTraversalFunctionTest extends FoundationTestCase {
+
+ private TimestampGranularityMonitor tsgm = new TimestampGranularityMonitor(BlazeClock.instance());
+ private RecordingEvaluationProgressReceiver progressReceiver;
+ private MemoizingEvaluator evaluator;
+ private SequentialBuildDriver driver;
+ private RecordingDifferencer differencer;
+ private AtomicReference<PathPackageLocator> pkgLocator;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+
+ pkgLocator = new AtomicReference<>(new PathPackageLocator(rootDirectory));
+ AtomicReference<ImmutableSet<PackageIdentifier>> deletedPackages =
+ new AtomicReference<>(ImmutableSet.<PackageIdentifier>of());
+ ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator);
+
+ Map<SkyFunctionName, SkyFunction> skyFunctions = new HashMap<>();
+
+ skyFunctions.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
+ skyFunctions.put(SkyFunctions.FILE, new FileFunction(pkgLocator, tsgm, externalFilesHelper));
+ skyFunctions.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction());
+ skyFunctions.put(
+ SkyFunctions.DIRECTORY_LISTING_STATE,
+ new DirectoryListingStateFunction(externalFilesHelper));
+ skyFunctions.put(
+ SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, new RecursiveFilesystemTraversalFunction());
+ skyFunctions.put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(deletedPackages));
+
+ progressReceiver = new RecordingEvaluationProgressReceiver();
+ differencer = new RecordingDifferencer();
+ evaluator = new InMemoryMemoizingEvaluator(skyFunctions, differencer, progressReceiver);
+ driver = new SequentialBuildDriver(evaluator);
+ PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
+ PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
+ }
+
+ private Artifact sourceArtifact(String path) {
+ return new Artifact(new PathFragment(path), Root.asSourceRoot(rootDirectory));
+ }
+
+ private Artifact sourceArtifactUnderPackagePath(String path, String packagePath) {
+ return new Artifact(
+ new PathFragment(path), Root.asSourceRoot(rootDirectory.getRelative(packagePath)));
+ }
+
+ private Artifact derivedArtifact(String path) {
+ PathFragment execPath = new PathFragment("out").getRelative(path);
+ Path fullPath = rootDirectory.getRelative(execPath);
+ Artifact output =
+ new Artifact(
+ fullPath,
+ Root.asDerivedRoot(rootDirectory, rootDirectory.getRelative("out")),
+ execPath);
+ return output;
+ }
+
+ private static RootedPath rootedPath(Artifact artifact) {
+ return RootedPath.toRootedPath(artifact.getRoot().getPath(), artifact.getRootRelativePath());
+ }
+
+ private RootedPath rootedPath(String path, String packagePath) {
+ return RootedPath.toRootedPath(rootDirectory.getRelative(packagePath), new PathFragment(path));
+ }
+
+ private static RootedPath childOf(Artifact artifact, String relative) {
+ return RootedPath.toRootedPath(
+ artifact.getRoot().getPath(), artifact.getRootRelativePath().getRelative(relative));
+ }
+
+ private static RootedPath childOf(RootedPath path, String relative) {
+ return RootedPath.toRootedPath(path.getRoot(), path.getRelativePath().getRelative(relative));
+ }
+
+ private static RootedPath parentOf(RootedPath path) {
+ PathFragment parent = Preconditions.checkNotNull(path.getRelativePath().getParentDirectory());
+ return RootedPath.toRootedPath(path.getRoot(), parent);
+ }
+
+ private static RootedPath siblingOf(RootedPath path, String relative) {
+ PathFragment parent = Preconditions.checkNotNull(path.getRelativePath().getParentDirectory());
+ return RootedPath.toRootedPath(path.getRoot(), parent.getRelative(relative));
+ }
+
+ private static RootedPath siblingOf(Artifact artifact, String relative) {
+ PathFragment parent =
+ Preconditions.checkNotNull(artifact.getRootRelativePath().getParentDirectory());
+ return RootedPath.toRootedPath(artifact.getRoot().getPath(), parent.getRelative(relative));
+ }
+
+ private void createFile(Path path, String... contents) throws Exception {
+ if (!path.getParentDirectory().exists()) {
+ scratch.dir(path.getParentDirectory().getPathString());
+ }
+ scratch.file(path.getPathString(), contents);
+ }
+
+ private void createFile(Artifact artifact, String... contents) throws Exception {
+ createFile(artifact.getPath(), contents);
+ }
+
+ private RootedPath createFile(RootedPath path, String... contents) throws Exception {
+ scratch.dir(parentOf(path).asPath().getPathString());
+ createFile(path.asPath(), contents);
+ return path;
+ }
+
+ private static TraversalRequest fileLikeRoot(Artifact file, PackageBoundaryMode pkgBoundaryMode) {
+ return new TraversalRequest(
+ rootedPath(file), !file.isSourceArtifact(), pkgBoundaryMode, false, null, null);
+ }
+
+ private static TraversalRequest pkgRoot(
+ RootedPath pkgDirectory, PackageBoundaryMode pkgBoundaryMode) {
+ return new TraversalRequest(pkgDirectory, false, pkgBoundaryMode, true, null, null);
+ }
+
+ private <T extends SkyValue> EvaluationResult<T> eval(SkyKey key) throws Exception {
+ return driver.evaluate(
+ ImmutableList.of(key),
+ false,
+ SkyframeExecutor.DEFAULT_THREAD_COUNT,
+ NullEventHandler.INSTANCE);
+ }
+
+ private RecursiveFilesystemTraversalValue evalTraversalRequest(TraversalRequest params)
+ throws Exception {
+ SkyKey key = rftvSkyKey(params);
+ EvaluationResult<RecursiveFilesystemTraversalValue> result = eval(key);
+ assertThat(result.hasError()).isFalse();
+ return result.get(key);
+ }
+
+ private static SkyKey rftvSkyKey(TraversalRequest params) {
+ return RecursiveFilesystemTraversalValue.key(params);
+ }
+
+ /**
+ * Asserts that the requested SkyValue can be built and results in the expected set of files.
+ *
+ * <p>The metadata of files is ignored in comparing the actual results with the expected ones.
+ * The returned object however contains the actual metadata.
+ */
+ @SafeVarargs
+ private final RecursiveFilesystemTraversalValue traverseAndAssertFiles(
+ TraversalRequest params, ResolvedFile... expectedFilesIgnoringMetadata) throws Exception {
+ Set<ResolvedFile> expectedMap = new HashSet<>();
+ for (ResolvedFile exp : expectedFilesIgnoringMetadata) {
+ // Strip metadata so only the type and path of the objects are compared.
+ expectedMap.add(exp.stripMetadataForTesting());
+ }
+ RecursiveFilesystemTraversalValue result = evalTraversalRequest(params);
+ Set<ResolvedFile> actualMap = new HashSet<>();
+ for (ResolvedFile act : result.getTransitiveFiles()) {
+ // Strip metadata so only the type and path of the objects are compared.
+ actualMap.add(act.stripMetadataForTesting());
+ }
+ // First just assert equality of the keys, so in case of a mismatch the error message is easier
+ // to read.
+ assertThat(expectedMap).isEqualTo(actualMap);
+
+ // The returned object still has the unstripped metadata.
+ return result;
+ }
+
+ private void appendToFile(RootedPath rootedPath, String content) throws Exception {
+ Path path = rootedPath.asPath();
+ if (path.exists()) {
+ try (OutputStream os = path.getOutputStream(/*append=*/ true)) {
+ os.write(content.getBytes(StandardCharsets.UTF_8));
+ }
+ differencer.invalidate(ImmutableList.of(FileStateValue.key(rootedPath)));
+ } else {
+ createFile(path, content);
+ }
+ }
+
+ private void appendToFile(Artifact file, String content) throws Exception {
+ appendToFile(rootedPath(file), content);
+ }
+
+ private void invalidateDirectory(RootedPath path) {
+ differencer.invalidate(ImmutableList.of(DirectoryListingStateValue.key(path)));
+ }
+
+ private void invalidateDirectory(Artifact directoryArtifact) {
+ invalidateDirectory(rootedPath(directoryArtifact));
+ }
+
+ private static final class RecordingEvaluationProgressReceiver
+ implements EvaluationProgressReceiver {
+ Set<SkyKey> invalidations;
+ Set<SkyValue> evaluations;
+
+ RecordingEvaluationProgressReceiver() {
+ clear();
+ }
+
+ void clear() {
+ invalidations = Sets.newConcurrentHashSet();
+ evaluations = Sets.newConcurrentHashSet();
+ }
+
+ @Override
+ public void invalidated(SkyKey skyKey, InvalidationState state) {
+ invalidations.add(skyKey);
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {}
+
+ @Override
+ public void computed(SkyKey skyKey, long elapsedTimeNanos) {}
+
+ @Override
+ public void evaluated(
+ SkyKey skyKey, Supplier<SkyValue> skyValueSupplier, EvaluationState state) {
+ SkyValue value = skyValueSupplier.get();
+ if (value != null) {
+ evaluations.add(value);
+ }
+ }
+ }
+
+ private ResolvedFile resolvedFile(RootedPath path) throws Exception {
+ return ResolvedFile.regularFile(path, FileStateValue.create(path, tsgm));
+ }
+
+ private ResolvedFile resolvedDanglingSymlink(RootedPath linkNamePath, PathFragment linkTargetPath)
+ throws Exception {
+ return ResolvedFile.danglingSymlink(
+ linkNamePath, linkTargetPath, FileStateValue.create(linkNamePath, tsgm));
+ }
+
+ private ResolvedFile resolvedSymlinkToFile(
+ RootedPath targetPath, RootedPath linkNamePath, PathFragment linkTargetPath)
+ throws Exception {
+ return ResolvedFile.symlinkToFile(
+ targetPath, linkNamePath, linkTargetPath, FileStateValue.create(linkNamePath, tsgm));
+ }
+
+ private ResolvedFile resolvedSymlinkToDir(
+ RootedPath targetPath, RootedPath linkNamePath, PathFragment linkTargetPath)
+ throws Exception {
+ return ResolvedFile.symlinkToDirectory(
+ targetPath, linkNamePath, linkTargetPath, FileStateValue.create(linkNamePath, tsgm));
+ }
+
+ private void assertTraversalOfFile(Artifact rootArtifact) throws Exception {
+ TraversalRequest traversalRoot = fileLikeRoot(rootArtifact, DONT_CROSS);
+ RootedPath rootedPath = createFile(rootedPath(rootArtifact), "foo");
+
+ // Assert that the SkyValue is built and looks right.
+ ResolvedFile expected = resolvedFile(rootedPath);
+ RecursiveFilesystemTraversalValue v1 = traverseAndAssertFiles(traversalRoot, expected);
+ assertThat(progressReceiver.invalidations).isEmpty();
+ assertThat(progressReceiver.evaluations).contains(v1);
+ progressReceiver.clear();
+
+ // Edit the file and verify that the value is rebuilt.
+ appendToFile(rootArtifact, "bar");
+ RecursiveFilesystemTraversalValue v2 = traverseAndAssertFiles(traversalRoot, expected);
+ assertThat(progressReceiver.invalidations).contains(rftvSkyKey(traversalRoot));
+ assertThat(progressReceiver.evaluations).contains(v2);
+ assertThat(v2).isNotEqualTo(v1);
+ progressReceiver.clear();
+ }
+
+ public void testTraversalOfSourceFile() throws Exception {
+ assertTraversalOfFile(sourceArtifact("foo/bar.txt"));
+ }
+
+ public void testTraversalOfGeneratedFile() throws Exception {
+ assertTraversalOfFile(derivedArtifact("foo/bar.txt"));
+ }
+
+ public void testTraversalOfSymlinkToFile() throws Exception {
+ Artifact linkNameArtifact = sourceArtifact("foo/baz/qux.sym");
+ Artifact linkTargetArtifact = sourceArtifact("foo/bar/baz.txt");
+ PathFragment linkValue = new PathFragment("../bar/baz.txt");
+ TraversalRequest traversalRoot = fileLikeRoot(linkNameArtifact, DONT_CROSS);
+ createFile(linkTargetArtifact);
+ scratch.dir(linkNameArtifact.getExecPath().getParentDirectory().getPathString());
+ rootDirectory.getRelative(linkNameArtifact.getExecPath()).createSymbolicLink(linkValue);
+
+ // Assert that the SkyValue is built and looks right.
+ RootedPath symlinkNamePath = rootedPath(linkNameArtifact);
+ RootedPath symlinkTargetPath = rootedPath(linkTargetArtifact);
+ ResolvedFile expected = resolvedSymlinkToFile(symlinkTargetPath, symlinkNamePath, linkValue);
+ RecursiveFilesystemTraversalValue v1 = traverseAndAssertFiles(traversalRoot, expected);
+ assertThat(progressReceiver.invalidations).isEmpty();
+ assertThat(progressReceiver.evaluations).contains(v1);
+ progressReceiver.clear();
+
+ // Edit the target of the symlink and verify that the value is rebuilt.
+ appendToFile(linkTargetArtifact, "bar");
+ RecursiveFilesystemTraversalValue v2 = traverseAndAssertFiles(traversalRoot, expected);
+ assertThat(progressReceiver.invalidations).contains(rftvSkyKey(traversalRoot));
+ assertThat(progressReceiver.evaluations).contains(v2);
+ assertThat(v2).isNotEqualTo(v1);
+ }
+
+ public void testTraversalOfTransitiveSymlinkToFile() throws Exception {
+ Artifact directLinkArtifact = sourceArtifact("direct/file.sym");
+ Artifact transitiveLinkArtifact = sourceArtifact("transitive/sym.sym");
+ RootedPath fileA = createFile(rootedPath(sourceArtifact("a/file.a")));
+ RootedPath directLink = rootedPath(directLinkArtifact);
+ RootedPath transitiveLink = rootedPath(transitiveLinkArtifact);
+ PathFragment directLinkPath = new PathFragment("../a/file.a");
+ PathFragment transitiveLinkPath = new PathFragment("../direct/file.sym");
+
+ parentOf(directLink).asPath().createDirectory();
+ parentOf(transitiveLink).asPath().createDirectory();
+ directLink.asPath().createSymbolicLink(directLinkPath);
+ transitiveLink.asPath().createSymbolicLink(transitiveLinkPath);
+
+ traverseAndAssertFiles(
+ fileLikeRoot(directLinkArtifact, DONT_CROSS),
+ resolvedSymlinkToFile(fileA, directLink, directLinkPath));
+
+ traverseAndAssertFiles(
+ fileLikeRoot(transitiveLinkArtifact, DONT_CROSS),
+ resolvedSymlinkToFile(fileA, transitiveLink, transitiveLinkPath));
+ }
+
+ private void assertTraversalOfDirectory(Artifact directoryArtifact) throws Exception {
+ // Create files under the directory.
+ // Use the root + root-relative path of the rootArtifact to create these files, rather than
+ // using the rootDirectory + execpath of the rootArtifact. The resulting paths are the same
+ // but the RootedPaths are different:
+ // in the 1st case, it is: RootedPath(/root/execroot, relative), in the second it is
+ // in the 2nd case, it is: RootedPath(/root, execroot/relative).
+ // Creating the files will also create the parent directories.
+ RootedPath file1 = createFile(childOf(directoryArtifact, "bar.txt"));
+ RootedPath file2 = createFile(childOf(directoryArtifact, "baz/qux.txt"));
+
+ TraversalRequest traversalRoot = fileLikeRoot(directoryArtifact, DONT_CROSS);
+
+ // Assert that the SkyValue is built and looks right.
+ ResolvedFile expected1 = resolvedFile(file1);
+ ResolvedFile expected2 = resolvedFile(file2);
+ RecursiveFilesystemTraversalValue v1 =
+ traverseAndAssertFiles(traversalRoot, expected1, expected2);
+ assertThat(progressReceiver.invalidations).isEmpty();
+ assertThat(progressReceiver.evaluations).contains(v1);
+ progressReceiver.clear();
+
+ // Add a new file to the directory and see that the value is rebuilt.
+ RootedPath file3 = createFile(childOf(directoryArtifact, "foo.txt"));
+ invalidateDirectory(directoryArtifact);
+ ResolvedFile expected3 = resolvedFile(file3);
+ RecursiveFilesystemTraversalValue v2 =
+ traverseAndAssertFiles(traversalRoot, expected1, expected2, expected3);
+ assertThat(progressReceiver.invalidations).contains(rftvSkyKey(traversalRoot));
+ assertThat(progressReceiver.evaluations).contains(v2);
+ assertThat(v2).isNotEqualTo(v1);
+ progressReceiver.clear();
+
+ // Edit a file in the directory and see that the value is rebuilt.
+ appendToFile(file1, "bar");
+ RecursiveFilesystemTraversalValue v3 =
+ traverseAndAssertFiles(traversalRoot, expected1, expected2, expected3);
+ assertThat(progressReceiver.invalidations).contains(rftvSkyKey(traversalRoot));
+ assertThat(progressReceiver.evaluations).contains(v3);
+ assertThat(v3).isNotEqualTo(v2);
+ progressReceiver.clear();
+
+ // Add a new file *outside* of the directory and see that the value is *not* rebuilt.
+ Artifact someFile = sourceArtifact("somewhere/else/a.file");
+ createFile(someFile, "new file");
+ appendToFile(someFile, "not all changes are treated equal");
+ RecursiveFilesystemTraversalValue v4 =
+ traverseAndAssertFiles(traversalRoot, expected1, expected2, expected3);
+ assertThat(v4).isEqualTo(v3);
+ assertThat(progressReceiver.invalidations).doesNotContain(rftvSkyKey(traversalRoot));
+ }
+
+ public void testTraversalOfSourceDirectory() throws Exception {
+ assertTraversalOfDirectory(sourceArtifact("dir"));
+ }
+
+ public void testTraversalOfGeneratedDirectory() throws Exception {
+ assertTraversalOfDirectory(derivedArtifact("dir"));
+ }
+
+ public void testTraversalOfTransitiveSymlinkToDirectory() throws Exception {
+ Artifact directLinkArtifact = sourceArtifact("direct/dir.sym");
+ Artifact transitiveLinkArtifact = sourceArtifact("transitive/sym.sym");
+ RootedPath fileA = createFile(rootedPath(sourceArtifact("a/file.a")));
+ RootedPath directLink = rootedPath(directLinkArtifact);
+ RootedPath transitiveLink = rootedPath(transitiveLinkArtifact);
+ PathFragment directLinkPath = new PathFragment("../a");
+ PathFragment transitiveLinkPath = new PathFragment("../direct/dir.sym");
+
+ parentOf(directLink).asPath().createDirectory();
+ parentOf(transitiveLink).asPath().createDirectory();
+ directLink.asPath().createSymbolicLink(directLinkPath);
+ transitiveLink.asPath().createSymbolicLink(transitiveLinkPath);
+
+ // Expect the file as if was a child of the direct symlink, not of the actual directory.
+ traverseAndAssertFiles(
+ fileLikeRoot(directLinkArtifact, DONT_CROSS),
+ resolvedSymlinkToDir(parentOf(fileA), directLink, directLinkPath),
+ resolvedFile(childOf(directLinkArtifact, "file.a")));
+
+ // Expect the file as if was a child of the transitive symlink, not of the actual directory.
+ traverseAndAssertFiles(
+ fileLikeRoot(transitiveLinkArtifact, DONT_CROSS),
+ resolvedSymlinkToDir(parentOf(fileA), transitiveLink, transitiveLinkPath),
+ resolvedFile(childOf(transitiveLinkArtifact, "file.a")));
+ }
+
+ public void testTraversePackage() throws Exception {
+ Artifact buildFile = sourceArtifact("pkg/BUILD");
+ RootedPath buildFilePath = createFile(rootedPath(buildFile));
+ RootedPath file1 = createFile(siblingOf(buildFile, "subdir/file.a"));
+
+ traverseAndAssertFiles(
+ pkgRoot(parentOf(buildFilePath), DONT_CROSS),
+ resolvedFile(buildFilePath),
+ resolvedFile(file1));
+ }
+
+ public void testTraversalOfSymlinkToDirectory() throws Exception {
+ Artifact linkNameArtifact = sourceArtifact("link/foo.sym");
+ Artifact linkTargetArtifact = sourceArtifact("dir");
+ RootedPath linkName = rootedPath(linkNameArtifact);
+ PathFragment linkValue = new PathFragment("../dir");
+ RootedPath file1 = createFile(childOf(linkTargetArtifact, "file.1"));
+ createFile(childOf(linkTargetArtifact, "sub/file.2"));
+ scratch.dir(parentOf(linkName).asPath().getPathString());
+ linkName.asPath().createSymbolicLink(linkValue);
+
+ // Assert that the SkyValue is built and looks right.
+ TraversalRequest traversalRoot = fileLikeRoot(linkNameArtifact, DONT_CROSS);
+ ResolvedFile expected1 =
+ resolvedSymlinkToDir(rootedPath(linkTargetArtifact), linkName, linkValue);
+ ResolvedFile expected2 = resolvedFile(childOf(linkNameArtifact, "file.1"));
+ ResolvedFile expected3 = resolvedFile(childOf(linkNameArtifact, "sub/file.2"));
+ // We expect to see all the files from the symlink'd directory, under the symlink's path, not
+ // under the symlink target's path.
+ RecursiveFilesystemTraversalValue v1 =
+ traverseAndAssertFiles(traversalRoot, expected1, expected2, expected3);
+ assertThat(progressReceiver.invalidations).isEmpty();
+ assertThat(progressReceiver.evaluations).contains(v1);
+ progressReceiver.clear();
+
+ // Add a new file to the directory and see that the value is rebuilt.
+ createFile(childOf(linkTargetArtifact, "file.3"));
+ invalidateDirectory(linkTargetArtifact);
+ ResolvedFile expected4 = resolvedFile(childOf(linkNameArtifact, "file.3"));
+ RecursiveFilesystemTraversalValue v2 =
+ traverseAndAssertFiles(traversalRoot, expected1, expected2, expected3, expected4);
+ assertThat(progressReceiver.invalidations).contains(rftvSkyKey(traversalRoot));
+ assertThat(progressReceiver.evaluations).contains(v2);
+ assertThat(v2).isNotEqualTo(v1);
+ progressReceiver.clear();
+
+ // Edit a file in the directory and see that the value is rebuilt.
+ appendToFile(file1, "bar");
+ RecursiveFilesystemTraversalValue v3 =
+ traverseAndAssertFiles(traversalRoot, expected1, expected2, expected3, expected4);
+ assertThat(progressReceiver.invalidations).contains(rftvSkyKey(traversalRoot));
+ assertThat(progressReceiver.evaluations).contains(v3);
+ assertThat(v3).isNotEqualTo(v2);
+ progressReceiver.clear();
+
+ // Add a new file *outside* of the directory and see that the value is *not* rebuilt.
+ Artifact someFile = sourceArtifact("somewhere/else/a.file");
+ createFile(someFile, "new file");
+ appendToFile(someFile, "not all changes are treated equal");
+ RecursiveFilesystemTraversalValue v4 =
+ traverseAndAssertFiles(traversalRoot, expected1, expected2, expected3, expected4);
+ assertThat(v4).isEqualTo(v3);
+ assertThat(progressReceiver.invalidations).doesNotContain(rftvSkyKey(traversalRoot));
+ }
+
+ public void testTraversalOfDanglingSymlink() throws Exception {
+ Artifact linkArtifact = sourceArtifact("a/dangling.sym");
+ RootedPath link = rootedPath(linkArtifact);
+ PathFragment linkTarget = new PathFragment("non_existent");
+ parentOf(link).asPath().createDirectory();
+ link.asPath().createSymbolicLink(linkTarget);
+ traverseAndAssertFiles(
+ fileLikeRoot(linkArtifact, DONT_CROSS), resolvedDanglingSymlink(link, linkTarget));
+ }
+
+ public void testTraversalOfDanglingSymlinkInADirectory() throws Exception {
+ Artifact dirArtifact = sourceArtifact("a");
+ RootedPath file = createFile(childOf(dirArtifact, "file.txt"));
+ RootedPath link = rootedPath(sourceArtifact("a/dangling.sym"));
+ PathFragment linkTarget = new PathFragment("non_existent");
+ parentOf(link).asPath().createDirectory();
+ link.asPath().createSymbolicLink(linkTarget);
+ traverseAndAssertFiles(
+ fileLikeRoot(dirArtifact, DONT_CROSS),
+ resolvedFile(file),
+ resolvedDanglingSymlink(link, linkTarget));
+ }
+
+ private void assertTraverseSubpackages(PackageBoundaryMode traverseSubpackages) throws Exception {
+ Artifact pkgDirArtifact = sourceArtifact("pkg1/foo");
+ Artifact subpkgDirArtifact = sourceArtifact("pkg1/foo/subdir/subpkg");
+ RootedPath pkgBuildFile = childOf(pkgDirArtifact, "BUILD");
+ RootedPath subpkgBuildFile = childOf(subpkgDirArtifact, "BUILD");
+ scratch.dir(rootedPath(pkgDirArtifact).asPath().getPathString());
+ scratch.dir(rootedPath(subpkgDirArtifact).asPath().getPathString());
+ createFile(pkgBuildFile);
+ createFile(subpkgBuildFile);
+
+ TraversalRequest traversalRoot = pkgRoot(parentOf(pkgBuildFile), traverseSubpackages);
+
+ ResolvedFile expected1 = resolvedFile(pkgBuildFile);
+ ResolvedFile expected2 = resolvedFile(subpkgBuildFile);
+ switch (traverseSubpackages) {
+ case CROSS:
+ traverseAndAssertFiles(traversalRoot, expected1, expected2);
+ break;
+ case DONT_CROSS:
+ traverseAndAssertFiles(traversalRoot, expected1);
+ break;
+ case REPORT_ERROR:
+ SkyKey key = rftvSkyKey(traversalRoot);
+ EvaluationResult<SkyValue> result = eval(key);
+ assertThat(result.hasError()).isTrue();
+ assertThat(result.getError().getException().getMessage())
+ .contains("crosses package boundary into package rooted at");
+ break;
+ default:
+ throw new IllegalStateException(traverseSubpackages.toString());
+ }
+ }
+
+ public void testTraverseSubpackages() throws Exception {
+ assertTraverseSubpackages(CROSS);
+ }
+
+ public void testDoNotTraverseSubpackages() throws Exception {
+ assertTraverseSubpackages(DONT_CROSS);
+ }
+
+ public void testReportErrorWhenTraversingSubpackages() throws Exception {
+ assertTraverseSubpackages(REPORT_ERROR);
+ }
+
+ public void testSwitchPackageRootsWhenUsingMultiplePackagePaths() throws Exception {
+ // Layout:
+ // pp1://a/BUILD
+ // pp1://a/file.a
+ // pp1://a/b.sym -> b/ (only created later)
+ // pp1://a/b/
+ // pp1://a/b/file.fake
+ // pp1://a/subdir/file.b
+ //
+ // pp2://a/BUILD
+ // pp2://a/b/
+ // pp2://a/b/BUILD
+ // pp2://a/b/file.a
+ // pp2://a/subdir.fake/
+ // pp2://a/subdir.fake/file.fake
+ //
+ // Notice that pp1://a/b will be overlaid by pp2://a/b as the latter has a BUILD file and that
+ // takes precedence. On the other hand the package definition pp2://a/BUILD will be ignored
+ // since package //a is already defined under pp1.
+ //
+ // Notice also that pp1://a/b.sym is a relative symlink pointing to b/. This should be resolved
+ // to the definition of //a/b/ under pp1, not under pp2.
+
+ // Set the package paths.
+ pkgLocator.set(
+ new PathPackageLocator(rootDirectory.getRelative("pp1"), rootDirectory.getRelative("pp2")));
+ PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
+
+ Artifact aBuildArtifact = sourceArtifactUnderPackagePath("a/BUILD", "pp1");
+ Artifact bBuildArtifact = sourceArtifactUnderPackagePath("a/b/BUILD", "pp2");
+
+ RootedPath pp1aBuild = createFile(rootedPath(aBuildArtifact));
+ RootedPath pp1aFileA = createFile(siblingOf(pp1aBuild, "file.a"));
+ RootedPath pp1bFileFake = createFile(siblingOf(pp1aBuild, "b/file.fake"));
+ RootedPath pp1aSubdirFileB = createFile(siblingOf(pp1aBuild, "subdir/file.b"));
+
+ RootedPath pp2aBuild = createFile(rootedPath("a/BUILD", "pp2"));
+ RootedPath pp2bBuild = createFile(rootedPath(bBuildArtifact));
+ RootedPath pp2bFileA = createFile(siblingOf(pp2bBuild, "file.a"));
+ createFile(siblingOf(pp2aBuild, "subdir.fake/file.fake"));
+
+ // Traverse //a including subpackages. The result should contain the pp1-definition of //a and
+ // the pp2-definition of //a/b.
+ traverseAndAssertFiles(
+ pkgRoot(parentOf(rootedPath(aBuildArtifact)), CROSS),
+ resolvedFile(pp1aBuild),
+ resolvedFile(pp1aFileA),
+ resolvedFile(pp1aSubdirFileB),
+ resolvedFile(pp2bBuild),
+ resolvedFile(pp2bFileA));
+
+ // Traverse //a excluding subpackages. The result should only contain files from //a and not
+ // from //a/b.
+ traverseAndAssertFiles(
+ pkgRoot(parentOf(rootedPath(aBuildArtifact)), DONT_CROSS),
+ resolvedFile(pp1aBuild),
+ resolvedFile(pp1aFileA),
+ resolvedFile(pp1aSubdirFileB));
+
+ // Create a relative symlink pp1://a/b.sym -> b/. It will be resolved to the subdirectory
+ // pp1://a/b, even though a package definition pp2://a/b exists.
+ RootedPath pp1aBsym = siblingOf(pp1aFileA, "b.sym");
+ pp1aBsym.asPath().createSymbolicLink(new PathFragment("b"));
+ invalidateDirectory(parentOf(pp1aBsym));
+
+ // Traverse //a excluding subpackages. The relative symlink //a/b.sym points to the subdirectory
+ // a/b, i.e. the pp1-definition, even though there is a pp2-defined package //a/b and we expect
+ // to see b.sym/b.fake (not b/b.fake).
+ traverseAndAssertFiles(
+ pkgRoot(parentOf(rootedPath(aBuildArtifact)), DONT_CROSS),
+ resolvedFile(pp1aBuild),
+ resolvedFile(pp1aFileA),
+ resolvedFile(childOf(pp1aBsym, "file.fake")),
+ resolvedSymlinkToDir(parentOf(pp1bFileFake), pp1aBsym, new PathFragment("b")),
+ resolvedFile(pp1aSubdirFileB));
+ }
+
+ public void testFileDigestChangeCausesRebuild() throws Exception {
+ Artifact artifact = sourceArtifact("foo/bar.txt");
+ RootedPath path = rootedPath(artifact);
+ createFile(path, "hello");
+
+ // Assert that the SkyValue is built and looks right.
+ TraversalRequest params = fileLikeRoot(artifact, DONT_CROSS);
+ ResolvedFile expected = resolvedFile(path);
+ RecursiveFilesystemTraversalValue v1 = traverseAndAssertFiles(params, expected);
+ assertThat(progressReceiver.evaluations).contains(v1);
+ progressReceiver.clear();
+
+ // Change the digest of the file. See that the value is rebuilt.
+ appendToFile(path, "world");
+ RecursiveFilesystemTraversalValue v2 = traverseAndAssertFiles(params, expected);
+ assertThat(progressReceiver.invalidations).contains(rftvSkyKey(params));
+ assertThat(v2).isNotEqualTo(v1);
+ }
+
+ public void testFileMtimeChangeDoesNotCauseRebuildIfDigestIsUnchanged() throws Exception {
+ Artifact artifact = sourceArtifact("foo/bar.txt");
+ RootedPath path = rootedPath(artifact);
+ createFile(path, "hello");
+
+ // Assert that the SkyValue is built and looks right.
+ TraversalRequest params = fileLikeRoot(artifact, DONT_CROSS);
+ ResolvedFile expected = resolvedFile(path);
+ RecursiveFilesystemTraversalValue v1 = traverseAndAssertFiles(params, expected);
+ assertThat(progressReceiver.evaluations).contains(v1);
+ progressReceiver.clear();
+
+ // Change the mtime of the file but not the digest. See that the value is *not* rebuilt.
+ long mtime = path.asPath().getLastModifiedTime();
+ mtime += 1000000L; // more than the timestamp granularity of any filesystem
+ path.asPath().setLastModifiedTime(mtime);
+ RecursiveFilesystemTraversalValue v2 = traverseAndAssertFiles(params, expected);
+ assertThat(v2).isEqualTo(v1);
+ }
+
+ public void testRegexp() throws Exception {
+ Artifact wantedArtifact = sourceArtifact("foo/bar/baz.txt");
+ Artifact unwantedArtifact = sourceArtifact("foo/boo/baztxt.bak");
+ RootedPath wantedPath = rootedPath(wantedArtifact);
+ createFile(wantedPath, "hello");
+ createFile(unwantedArtifact, "nope");
+ Artifact pkgDirArtifact = sourceArtifact("foo");
+ RootedPath dir = rootedPath(pkgDirArtifact);
+ scratch.dir(dir.asPath().getPathString());
+
+ TraversalRequest traversalRoot =
+ new TraversalRequest(
+ dir, false, PackageBoundaryMode.REPORT_ERROR, true, null, Pattern.compile(".*\\.txt"));
+
+ ResolvedFile expected = resolvedFile(wantedPath);
+ traverseAndAssertFiles(traversalRoot, expected);
+ }
+
+ public void testGeneratedDirectoryConflictsWithPackage() throws Exception {
+ Artifact genDir = derivedArtifact("a/b");
+ createFile(rootedPath(sourceArtifact("a/b/c/file.real")));
+ createFile(rootedPath(derivedArtifact("a/b/c/file.fake")));
+ createFile(sourceArtifact("a/b/c/BUILD"));
+
+ SkyKey key = rftvSkyKey(fileLikeRoot(genDir, CROSS));
+ EvaluationResult<SkyValue> result = eval(key);
+ assertThat(result.hasError()).isTrue();
+ ErrorInfo error = result.getError(key);
+ assertThat(error.isTransient()).isFalse();
+ assertThat(error.getException().getMessage())
+ .contains("Generated directory a/b/c conflicts with package under the same path.");
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunctionTest.java
new file mode 100644
index 0000000..ce2b84d
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/RecursivePkgFunctionTest.java
@@ -0,0 +1,169 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.BuildDriver;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.WalkableGraph;
+
+/**
+ * Tests for {@link RecursivePkgFunction}. Unfortunately, we can't directly test
+ * RecursivePkgFunction as it uses PackageValues, and PackageFunction uses legacy stuff that
+ * isn't easily mockable. So our testing strategy is to make hacky calls to
+ * SequencedSkyframeExecutor.
+ *
+ * <p>Target parsing tests already cover most of the behavior of RecursivePkgFunction, but there
+ * are a couple of corner cases we need to test directly.
+ */
+public class RecursivePkgFunctionTest extends BuildViewTestCase {
+
+ private SkyframeExecutor skyframeExecutor;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ skyframeExecutor = getSkyframeExecutor();
+ }
+
+ private SkyKey buildRecursivePkgKey(
+ Path root, PathFragment rootRelativePath, ImmutableSet<PathFragment> excludedPaths) {
+ RootedPath rootedPath = RootedPath.toRootedPath(root, rootRelativePath);
+ return RecursivePkgValue.key(
+ PackageIdentifier.DEFAULT_REPOSITORY_NAME, rootedPath, excludedPaths);
+ }
+
+ private RecursivePkgValue buildRecursivePkgValue(Path root, PathFragment rootRelativePath)
+ throws Exception {
+ return buildRecursivePkgValue(root, rootRelativePath, ImmutableSet.<PathFragment>of());
+ }
+
+ private RecursivePkgValue buildRecursivePkgValue(
+ Path root, PathFragment rootRelativePath, ImmutableSet<PathFragment> excludedPaths)
+ throws Exception {
+ SkyKey key = buildRecursivePkgKey(root, rootRelativePath, excludedPaths);
+ return getEvaluationResult(key).get(key);
+ }
+
+ private EvaluationResult<RecursivePkgValue> getEvaluationResult(SkyKey key)
+ throws InterruptedException {
+ BuildDriver driver = skyframeExecutor.getDriverForTesting();
+ EvaluationResult<RecursivePkgValue> evaluationResult =
+ driver.evaluate(
+ ImmutableList.of(key),
+ /*keepGoing=*/ false,
+ SequencedSkyframeExecutor.DEFAULT_THREAD_COUNT,
+ reporter);
+ Preconditions.checkState(!evaluationResult.hasError());
+ return evaluationResult;
+ }
+
+ public void testStartingAtBuildFile() throws Exception {
+ scratch.file("a/b/c/BUILD");
+ RecursivePkgValue value =
+ buildRecursivePkgValue(rootDirectory, new PathFragment("a/b/c/BUILD"));
+ assertTrue(value.getPackages().isEmpty());
+ }
+
+ public void testPackagesUnderMultipleRoots() throws Exception {
+ Path root1 = rootDirectory.getRelative("root1");
+ Path root2 = rootDirectory.getRelative("root2");
+ scratch.file(root1 + "/WORKSPACE");
+ scratch.file(root2 + "/WORKSPACE");
+ scratch.file(root1 + "/a/BUILD");
+ scratch.file(root2 + "/a/b/BUILD");
+ setPackageCacheOptions("--package_path=" + "root1" + ":" + "root2");
+
+ RecursivePkgValue valueForRoot1 = buildRecursivePkgValue(root1, new PathFragment("a"));
+ String root1Pkg = Iterables.getOnlyElement(valueForRoot1.getPackages());
+ assertEquals(root1Pkg, "a");
+
+ RecursivePkgValue valueForRoot2 = buildRecursivePkgValue(root2, new PathFragment("a"));
+ String root2Pkg = Iterables.getOnlyElement(valueForRoot2.getPackages());
+ assertEquals(root2Pkg, "a/b");
+ }
+
+ public void testSubdirectoryExclusion() throws Exception {
+ // Given a package "a" with two packages below it, "a/b" and "a/c",
+ scratch.file("a/BUILD");
+ scratch.file("a/b/BUILD");
+ scratch.file("a/c/BUILD");
+
+ // When the top package is evaluated for recursive package values, and "a/b" is excluded,
+ PathFragment excludedPathFragment = new PathFragment("a/b");
+ SkyKey key =
+ buildRecursivePkgKey(
+ rootDirectory, new PathFragment("a"), ImmutableSet.of(excludedPathFragment));
+ EvaluationResult<RecursivePkgValue> evaluationResult = getEvaluationResult(key);
+ RecursivePkgValue value = evaluationResult.get(key);
+
+ // Then the package corresponding to "a/b" is not present in the result,
+ assertThat(value.getPackages()).doesNotContain("a/b");
+
+ // And the "a" package and "a/c" package are.
+ assertThat(value.getPackages()).contains("a");
+ assertThat(value.getPackages()).contains("a/c");
+
+ // Also, the computation graph does not contain a cached value for "a/b".
+ WalkableGraph graph = Preconditions.checkNotNull(evaluationResult.getWalkableGraph());
+ assertFalse(
+ graph.exists(
+ buildRecursivePkgKey(
+ rootDirectory, excludedPathFragment, ImmutableSet.<PathFragment>of())));
+
+ // And the computation graph does contain a cached value for "a/c" with the empty set excluded,
+ // because that key was evaluated.
+ assertTrue(
+ graph.exists(
+ buildRecursivePkgKey(
+ rootDirectory, new PathFragment("a/c"), ImmutableSet.<PathFragment>of())));
+ }
+
+ public void testExcludedSubdirectoryGettingPassedDown() throws Exception {
+ // Given a package "a" with two packages below a directory below it, "a/b/c" and "a/b/d",
+ scratch.file("a/BUILD");
+ scratch.file("a/b/c/BUILD");
+ scratch.file("a/b/d/BUILD");
+
+ // When the top package is evaluated for recursive package values, and "a/b/c" is excluded,
+ ImmutableSet<PathFragment> excludedPaths = ImmutableSet.of(new PathFragment("a/b/c"));
+ SkyKey key = buildRecursivePkgKey(rootDirectory, new PathFragment("a"), excludedPaths);
+ EvaluationResult<RecursivePkgValue> evaluationResult = getEvaluationResult(key);
+ RecursivePkgValue value = evaluationResult.get(key);
+
+ // Then the package corresponding to the excluded subdirectory is not present in the result,
+ assertThat(value.getPackages()).doesNotContain("a/b/c");
+
+ // And the top package and other subsubdirectory package are.
+ assertThat(value.getPackages()).contains("a");
+ assertThat(value.getPackages()).contains("a/b/d");
+
+ // Also, the computation graph contains a cached value for "a/b" with "a/b/c" excluded, because
+ // "a/b/c" does live underneath "a/b".
+ WalkableGraph graph = Preconditions.checkNotNull(evaluationResult.getWalkableGraph());
+ assertTrue(
+ graph.exists(buildRecursivePkgKey(rootDirectory, new PathFragment("a/b"), excludedPaths)));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RecursivePkgKeyTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RecursivePkgKeyTest.java
new file mode 100644
index 0000000..4c312db
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/RecursivePkgKeyTest.java
@@ -0,0 +1,81 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.skyframe.RecursivePkgValue.RecursivePkgKey;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.SkyKey;
+
+/** Tests for {@link RecursivePkgKey}. */
+public class RecursivePkgKeyTest extends BuildViewTestCase {
+
+ private SkyKey buildRecursivePkgKey(
+ RepositoryName repository,
+ PathFragment rootRelativePath,
+ ImmutableSet<PathFragment> excludedPaths) {
+ RootedPath rootedPath = RootedPath.toRootedPath(rootDirectory, rootRelativePath);
+ return RecursivePkgValue.key(repository, rootedPath, excludedPaths);
+ }
+
+ private void invalidHelper(
+ PathFragment rootRelativePath, ImmutableSet<PathFragment> excludedPaths) {
+ try {
+ buildRecursivePkgKey(
+ PackageIdentifier.DEFAULT_REPOSITORY_NAME, rootRelativePath, excludedPaths);
+ fail();
+ } catch (IllegalArgumentException expected) {
+ }
+ }
+
+ public void testValidRecursivePkgKeys() throws Exception {
+ buildRecursivePkgKey(
+ PackageIdentifier.DEFAULT_REPOSITORY_NAME,
+ new PathFragment(""),
+ ImmutableSet.<PathFragment>of());
+ buildRecursivePkgKey(
+ PackageIdentifier.DEFAULT_REPOSITORY_NAME,
+ new PathFragment(""),
+ ImmutableSet.of(new PathFragment("a")));
+
+ buildRecursivePkgKey(
+ PackageIdentifier.DEFAULT_REPOSITORY_NAME,
+ new PathFragment("a"),
+ ImmutableSet.<PathFragment>of());
+ buildRecursivePkgKey(
+ PackageIdentifier.DEFAULT_REPOSITORY_NAME,
+ new PathFragment("a"),
+ ImmutableSet.of(new PathFragment("a/b")));
+
+ buildRecursivePkgKey(
+ PackageIdentifier.DEFAULT_REPOSITORY_NAME,
+ new PathFragment("a/b"),
+ ImmutableSet.<PathFragment>of());
+ buildRecursivePkgKey(
+ PackageIdentifier.DEFAULT_REPOSITORY_NAME,
+ new PathFragment("a/b"),
+ ImmutableSet.of(new PathFragment("a/b/c")));
+ }
+
+ public void testInvalidRecursivePkgKeys() throws Exception {
+ invalidHelper(new PathFragment(""), ImmutableSet.of(new PathFragment("")));
+ invalidHelper(new PathFragment("a"), ImmutableSet.of(new PathFragment("a")));
+ invalidHelper(new PathFragment("a"), ImmutableSet.of(new PathFragment("b")));
+ invalidHelper(new PathFragment("a/b"), ImmutableSet.of(new PathFragment("a")));
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/SkyframeAwareActionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/SkyframeAwareActionTest.java
new file mode 100644
index 0000000..282bb51
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/SkyframeAwareActionTest.java
@@ -0,0 +1,786 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Callables;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
+import com.google.devtools.build.lib.actions.util.DummyExecutor;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
+import com.google.devtools.build.skyframe.EvaluationProgressReceiver.EvaluationState;
+import com.google.devtools.build.skyframe.SkyFunction.Environment;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.annotation.Nullable;
+
+/** Tests for {@link SkyframeAwareAction}. */
+public class SkyframeAwareActionTest extends TimestampBuilderTestCase {
+ private Builder builder;
+ private Executor executor;
+ private TrackingEvaluationProgressReceiver invalidationReceiver;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ invalidationReceiver = new TrackingEvaluationProgressReceiver();
+ builder = createBuilder(inMemoryCache, 1, /*keepGoing=*/ false, invalidationReceiver);
+ executor = new DummyExecutor(rootDirectory);
+ }
+
+ private static final class TrackingEvaluationProgressReceiver
+ implements EvaluationProgressReceiver {
+
+ public static final class InvalidatedKey {
+ public final SkyKey skyKey;
+ public final InvalidationState state;
+
+ InvalidatedKey(SkyKey skyKey, InvalidationState state) {
+ this.skyKey = skyKey;
+ this.state = state;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof InvalidatedKey
+ && this.skyKey.equals(((InvalidatedKey) obj).skyKey)
+ && this.state.equals(((InvalidatedKey) obj).state);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(skyKey, state);
+ }
+ }
+
+ public static final class EvaluatedEntry {
+ public final SkyKey skyKey;
+ public final SkyValue value;
+ public final EvaluationState state;
+
+ EvaluatedEntry(SkyKey skyKey, SkyValue value, EvaluationState state) {
+ this.skyKey = skyKey;
+ this.value = value;
+ this.state = state;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof EvaluatedEntry
+ && this.skyKey.equals(((EvaluatedEntry) obj).skyKey)
+ && this.value.equals(((EvaluatedEntry) obj).value)
+ && this.state.equals(((EvaluatedEntry) obj).state);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(skyKey, value, state);
+ }
+ }
+
+ public final Set<InvalidatedKey> invalidated = Sets.newConcurrentHashSet();
+ public final Set<SkyKey> enqueued = Sets.newConcurrentHashSet();
+ public final Set<EvaluatedEntry> evaluated = Sets.newConcurrentHashSet();
+
+ public void reset() {
+ invalidated.clear();
+ enqueued.clear();
+ evaluated.clear();
+ }
+
+ public boolean wasInvalidated(SkyKey skyKey) {
+ for (InvalidatedKey e : invalidated) {
+ if (e.skyKey.equals(skyKey)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public EvaluatedEntry getEvalutedEntry(SkyKey forKey) {
+ for (EvaluatedEntry e : evaluated) {
+ if (e.skyKey.equals(forKey)) {
+ return e;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public void invalidated(SkyKey skyKey, InvalidationState state) {
+ invalidated.add(new InvalidatedKey(skyKey, state));
+ }
+
+ @Override
+ public void enqueueing(SkyKey skyKey) {
+ enqueued.add(skyKey);
+ }
+
+ @Override
+ public void computed(SkyKey skyKey, long elapsedTimeNanos) {}
+
+ @Override
+ public void evaluated(
+ SkyKey skyKey, Supplier<SkyValue> skyValueSupplier, EvaluationState state) {
+ evaluated.add(new EvaluatedEntry(skyKey, skyValueSupplier.get(), state));
+ }
+ }
+
+ /** A mock action that counts how many times it was executed. */
+ private static class ExecutionCountingAction extends AbstractAction {
+ private final AtomicInteger executionCounter;
+
+ ExecutionCountingAction(Artifact input, Artifact output, AtomicInteger executionCounter) {
+ super(ActionsTestUtil.NULL_ACTION_OWNER, ImmutableList.of(input), ImmutableList.of(output));
+ this.executionCounter = executionCounter;
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ executionCounter.incrementAndGet();
+
+ // This action first reads its input file (there can be only one). For the purpose of these
+ // tests we assume that the input file is short, maybe just 10 bytes long.
+ byte[] input = new byte[10];
+ int inputLen = 0;
+ try (InputStream in = Iterables.getOnlyElement(getInputs()).getPath().getInputStream()) {
+ inputLen = in.read(input);
+ } catch (IOException e) {
+ throw new ActionExecutionException(e, this, false);
+ }
+
+ // This action then writes the contents of the input to the (only) output file, and appends an
+ // extra "x" character too.
+ try (OutputStream out = getPrimaryOutput().getPath().getOutputStream()) {
+ out.write(input, 0, inputLen);
+ out.write('x');
+ } catch (IOException e) {
+ throw new ActionExecutionException(e, this, false);
+ }
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return null;
+ }
+
+ @Override
+ public String getMnemonic() {
+ return null;
+ }
+
+ @Override
+ protected String computeKey() {
+ return getPrimaryOutput().getExecPathString() + executionCounter.get();
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return ResourceSet.ZERO;
+ }
+ }
+
+ private static class ExecutionCountingCacheBypassingAction extends ExecutionCountingAction {
+ ExecutionCountingCacheBypassingAction(
+ Artifact input, Artifact output, AtomicInteger executionCounter) {
+ super(input, output, executionCounter);
+ }
+
+ @Override
+ public boolean executeUnconditionally() {
+ return true;
+ }
+
+ @Override
+ public boolean isVolatile() {
+ return true;
+ }
+ }
+
+ /** A mock skyframe-aware action that counts how many times it was executed. */
+ private static class SkyframeAwareExecutionCountingAction
+ extends ExecutionCountingCacheBypassingAction implements SkyframeAwareAction {
+ private final SkyKey actionDepKey;
+
+ SkyframeAwareExecutionCountingAction(
+ Artifact input, Artifact output, AtomicInteger executionCounter, SkyKey actionDepKey) {
+ super(input, output, executionCounter);
+ this.actionDepKey = actionDepKey;
+ }
+
+ @Override
+ public void establishSkyframeDependencies(Environment env) throws ExceptionBase {
+ // Establish some Skyframe dependency. A real action would then use this to compute and
+ // cache data for the execute(...) method.
+ env.getValue(actionDepKey);
+ }
+ }
+
+ private interface ExecutionCountingActionFactory {
+ ExecutionCountingAction create(Artifact input, Artifact output, AtomicInteger executionCounter);
+ }
+
+ private enum ChangeArtifact {
+ DONT_CHANGE,
+ CHANGE_MTIME {
+ @Override
+ boolean changeMtime() {
+ return true;
+ }
+ },
+ CHANGE_MTIME_AND_CONTENT {
+ @Override
+ boolean changeMtime() {
+ return true;
+ }
+
+ @Override
+ boolean changeContent() {
+ return true;
+ }
+ };
+
+ boolean changeMtime() {
+ return false;
+ }
+
+ boolean changeContent() {
+ return false;
+ }
+ }
+
+ private enum ExpectActionIs {
+ NOT_DIRTIED {
+ @Override
+ boolean actuallyClean() {
+ return true;
+ }
+ },
+ DIRTIED_BUT_VERIFIED_CLEAN {
+ @Override
+ boolean dirtied() {
+ return true;
+ }
+
+ @Override
+ boolean actuallyClean() {
+ return true;
+ }
+ },
+
+ // REBUILT_BUT_ACTION_CACHE_HIT,
+ // This would be a bug, symptom of a skyframe-aware action that doesn't bypass the action cache
+ // and is incorrectly regarded as an action cache hit when its inputs stayed the same but its
+ // "skyframe dependencies" changed.
+
+ REEXECUTED {
+ @Override
+ boolean dirtied() {
+ return true;
+ }
+
+ @Override
+ boolean reexecuted() {
+ return true;
+ }
+ };
+
+ boolean dirtied() {
+ return false;
+ }
+
+ boolean actuallyClean() {
+ return false;
+ }
+
+ boolean reexecuted() {
+ return false;
+ }
+ }
+
+ private void maybeChangeFile(Artifact file, ChangeArtifact changeRequest) throws Exception {
+ if (changeRequest == ChangeArtifact.DONT_CHANGE) {
+ return;
+ }
+
+ if (changeRequest.changeMtime()) {
+ // 1000000 should be larger than the filesystem timestamp granularity.
+ file.getPath().setLastModifiedTime(file.getPath().getLastModifiedTime() + 1000000);
+ tsgm.waitForTimestampGranularity(reporter.getOutErr());
+ }
+
+ if (changeRequest.changeContent()) {
+ appendToFile(file.getPath());
+ }
+
+ // Invalidate the file state value to inform Skyframe that the file may have changed.
+ // This will also invalidate the action execution value.
+ differencer.invalidate(
+ ImmutableList.of(
+ FileStateValue.key(
+ RootedPath.toRootedPath(file.getRoot().getPath(), file.getRootRelativePath()))));
+ }
+
+ private void assertActionExecutions(
+ ExecutionCountingActionFactory actionFactory,
+ ChangeArtifact changeActionInput,
+ Callable<Void> betweenBuilds,
+ ExpectActionIs expectActionIs)
+ throws Exception {
+ // Set up the action's input, output, owner and most importantly the execution counter.
+ Artifact actionInput = createSourceArtifact("foo/action-input.txt");
+ Artifact actionOutput = createDerivedArtifact("foo/action-output.txt");
+ AtomicInteger executionCounter = new AtomicInteger(0);
+
+ scratch.file(actionInput.getPath().getPathString(), "foo");
+
+ // Generating actions of artifacts are found by looking them up in the graph. The lookup value
+ // must be present in the graph before execution.
+ Action action = actionFactory.create(actionInput, actionOutput, executionCounter);
+ registerAction(action);
+
+ // Build the output for the first time.
+ builder.buildArtifacts(
+ reporter,
+ ImmutableSet.of(actionOutput),
+ null,
+ null,
+ null,
+ null,
+ executor,
+ null,
+ false,
+ null);
+
+ // Sanity check that our invalidation receiver is working correctly. We'll rely on it again.
+ SkyKey actionKey = ActionExecutionValue.key(action);
+ TrackingEvaluationProgressReceiver.EvaluatedEntry evaluatedAction =
+ invalidationReceiver.getEvalutedEntry(actionKey);
+ assertThat(evaluatedAction).isNotNull();
+ SkyValue actionValue = evaluatedAction.value;
+
+ // Mutate the action input if requested.
+ maybeChangeFile(actionInput, changeActionInput);
+
+ // Execute user code before next build.
+ betweenBuilds.call();
+
+ // Rebuild the output.
+ invalidationReceiver.reset();
+ builder.buildArtifacts(
+ reporter,
+ ImmutableSet.of(actionOutput),
+ null,
+ null,
+ null,
+ null,
+ executor,
+ null,
+ false,
+ null);
+
+ if (expectActionIs.dirtied()) {
+ assertThat(invalidationReceiver.wasInvalidated(actionKey)).isTrue();
+
+ TrackingEvaluationProgressReceiver.EvaluatedEntry newEntry =
+ invalidationReceiver.getEvalutedEntry(actionKey);
+ assertThat(newEntry).isNotNull();
+ if (expectActionIs.actuallyClean()) {
+ // Action was dirtied but verified clean.
+ assertThat(newEntry.state).isEqualTo(EvaluationState.CLEAN);
+ assertThat(newEntry.value).isEqualTo(actionValue);
+ } else {
+ // Action was dirtied and rebuilt. It was either reexecuted or was an action cache hit,
+ // doesn't matter here.
+ assertThat(newEntry.state).isEqualTo(EvaluationState.BUILT);
+ assertThat(newEntry.value).isNotEqualTo(actionValue);
+ }
+ } else {
+ // Action was not dirtied.
+ assertThat(invalidationReceiver.wasInvalidated(actionKey)).isFalse();
+ }
+
+ // Assert that the action was executed the right number of times. Whether the action execution
+ // function was called again is up for the test method to verify.
+ assertThat(executionCounter.get()).isEqualTo(expectActionIs.reexecuted() ? 2 : 1);
+ }
+
+ private RootedPath createSkyframeDepOfAction() throws Exception {
+ scratch.file(rootDirectory.getRelative("action.dep").getPathString(), "blah");
+ return RootedPath.toRootedPath(rootDirectory, new PathFragment("action.dep"));
+ }
+
+ private void appendToFile(Path path) throws Exception {
+ try (OutputStream stm = path.getOutputStream(/*append=*/ true)) {
+ stm.write("blah".getBytes(StandardCharsets.UTF_8));
+ }
+ }
+
+ public void testCacheCheckingActionWithContentChangingInput() throws Exception {
+ assertActionWithContentChangingInput(/* unconditionalExecution */ false);
+ }
+
+ public void testCacheBypassingActionWithContentChangingInput() throws Exception {
+ assertActionWithContentChangingInput(/* unconditionalExecution */ true);
+ }
+
+ private void assertActionWithContentChangingInput(final boolean unconditionalExecution)
+ throws Exception {
+ // Assert that a simple, non-skyframe-aware action is executed twice
+ // if its input's content changes between builds.
+ assertActionExecutions(
+ new ExecutionCountingActionFactory() {
+ @Override
+ public ExecutionCountingAction create(
+ Artifact input, Artifact output, AtomicInteger executionCounter) {
+ return unconditionalExecution
+ ? new ExecutionCountingCacheBypassingAction(input, output, executionCounter)
+ : new ExecutionCountingAction(input, output, executionCounter);
+ }
+ },
+ ChangeArtifact.CHANGE_MTIME_AND_CONTENT,
+ Callables.<Void>returning(null),
+ ExpectActionIs.REEXECUTED);
+ }
+
+ public void testCacheCheckingActionWithMtimeChangingInput() throws Exception {
+ assertActionWithMtimeChangingInput(/* unconditionalExecution */ false);
+ }
+
+ public void testCacheBypassingActionWithMtimeChangingInput() throws Exception {
+ assertActionWithMtimeChangingInput(/* unconditionalExecution */ true);
+ }
+
+ private void assertActionWithMtimeChangingInput(final boolean unconditionalExecution)
+ throws Exception {
+ // Assert that a simple, non-skyframe-aware action is executed only once
+ // if its input's mtime changes but its contents stay the same between builds.
+ assertActionExecutions(
+ new ExecutionCountingActionFactory() {
+ @Override
+ public ExecutionCountingAction create(
+ Artifact input, Artifact output, AtomicInteger executionCounter) {
+ return unconditionalExecution
+ ? new ExecutionCountingCacheBypassingAction(input, output, executionCounter)
+ : new ExecutionCountingAction(input, output, executionCounter);
+ }
+ },
+ ChangeArtifact.CHANGE_MTIME,
+ Callables.<Void>returning(null),
+ ExpectActionIs.DIRTIED_BUT_VERIFIED_CLEAN);
+ }
+
+ public void testActionWithNonChangingInput(final boolean unconditionalExecution)
+ throws Exception {
+ // Assert that a simple, non-skyframe-aware action is executed only once
+ // if its input does not change at all between builds.
+ assertActionExecutions(
+ new ExecutionCountingActionFactory() {
+ @Override
+ public ExecutionCountingAction create(
+ Artifact input, Artifact output, AtomicInteger executionCounter) {
+ return unconditionalExecution
+ ? new ExecutionCountingCacheBypassingAction(input, output, executionCounter)
+ : new ExecutionCountingAction(input, output, executionCounter);
+ }
+ },
+ ChangeArtifact.DONT_CHANGE,
+ Callables.<Void>returning(null),
+ ExpectActionIs.NOT_DIRTIED);
+ }
+
+ private void assertActionWithMaybeChangingInputAndChangingSkyframeDeps(
+ ChangeArtifact changeInputFile) throws Exception {
+ final RootedPath depPath = createSkyframeDepOfAction();
+ final SkyKey skyframeDep = FileStateValue.key(depPath);
+
+ // Assert that an action-cache-check-bypassing action is executed twice if its skyframe deps
+ // change while its input does not. The skyframe dependency is established by making the action
+ // skyframe-aware and updating the value between builds.
+ assertActionExecutions(
+ new ExecutionCountingActionFactory() {
+ @Override
+ public ExecutionCountingAction create(
+ Artifact input, Artifact output, AtomicInteger executionCounter) {
+ return new SkyframeAwareExecutionCountingAction(
+ input, output, executionCounter, skyframeDep);
+ }
+ },
+ changeInputFile,
+ new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ // Invalidate the dependency and change what its value will be in the next build. This
+ // should enforce rebuilding of the action.
+ appendToFile(depPath.asPath());
+ differencer.invalidate(ImmutableList.of(skyframeDep));
+ return null;
+ }
+ },
+ ExpectActionIs.REEXECUTED);
+ }
+
+ public void testActionWithNonChangingInputButChangingSkyframeDeps() throws Exception {
+ assertActionWithMaybeChangingInputAndChangingSkyframeDeps(ChangeArtifact.DONT_CHANGE);
+ }
+
+ public void testActionWithChangingInputMtimeAndChangingSkyframeDeps() throws Exception {
+ assertActionWithMaybeChangingInputAndChangingSkyframeDeps(ChangeArtifact.CHANGE_MTIME);
+ }
+
+ public void testActionWithChangingInputAndChangingSkyframeDeps() throws Exception {
+ assertActionWithMaybeChangingInputAndChangingSkyframeDeps(
+ ChangeArtifact.CHANGE_MTIME_AND_CONTENT);
+ }
+
+ public void testActionWithNonChangingInputAndNonChangingSkyframeDeps() throws Exception {
+ final SkyKey skyframeDep = FileStateValue.key(createSkyframeDepOfAction());
+
+ // Assert that an action-cache-check-bypassing action is executed only once if neither its input
+ // nor its Skyframe dependency changes between builds.
+ assertActionExecutions(
+ new ExecutionCountingActionFactory() {
+ @Override
+ public ExecutionCountingAction create(
+ Artifact input, Artifact output, AtomicInteger executionCounter) {
+ return new SkyframeAwareExecutionCountingAction(
+ input, output, executionCounter, skyframeDep);
+ }
+ },
+ ChangeArtifact.DONT_CHANGE,
+ new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ // Invalidate the dependency but leave its value up-to-date, so the action should not
+ // be rebuilt.
+ differencer.invalidate(ImmutableList.of(skyframeDep));
+ return null;
+ }
+ },
+ ExpectActionIs.DIRTIED_BUT_VERIFIED_CLEAN);
+ }
+
+ private abstract static class SingleOutputAction extends AbstractAction {
+ SingleOutputAction(@Nullable Artifact input, Artifact output) {
+ super(
+ ActionsTestUtil.NULL_ACTION_OWNER,
+ input == null ? ImmutableList.<Artifact>of() : ImmutableList.of(input),
+ ImmutableList.of(output));
+ }
+
+ protected static final class Buffer {
+ final int size;
+ final byte[] data;
+
+ Buffer(byte[] data, int size) {
+ this.data = data;
+ this.size = size;
+ }
+ }
+
+ protected Buffer readInput() throws ActionExecutionException {
+ byte[] input = new byte[100];
+ int inputLen = 0;
+ try (InputStream in = getPrimaryInput().getPath().getInputStream()) {
+ inputLen = in.read(input, 0, input.length);
+ } catch (IOException e) {
+ throw new ActionExecutionException(e, this, false);
+ }
+ return new Buffer(input, inputLen);
+ }
+
+ protected void writeOutput(@Nullable Buffer buf, String data) throws ActionExecutionException {
+ try (OutputStream out = getPrimaryOutput().getPath().getOutputStream()) {
+ if (buf != null) {
+ out.write(buf.data, 0, buf.size);
+ }
+ out.write(data.getBytes(StandardCharsets.UTF_8), 0, data.length());
+ } catch (IOException e) {
+ throw new ActionExecutionException(e, this, false);
+ }
+ }
+
+ @Override
+ public String describeStrategy(Executor executor) {
+ return null;
+ }
+
+ @Override
+ public String getMnemonic() {
+ return "MockActionMnemonic";
+ }
+
+ @Override
+ protected String computeKey() {
+ return new Fingerprint().addInt(42).hexDigestAndReset();
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(Executor executor) {
+ return ResourceSet.ZERO;
+ }
+ }
+
+ private abstract static class SingleOutputSkyframeAwareAction extends SingleOutputAction
+ implements SkyframeAwareAction {
+ SingleOutputSkyframeAwareAction(@Nullable Artifact input, Artifact output) {
+ super(input, output);
+ }
+
+ @Override
+ public boolean executeUnconditionally() {
+ return true;
+ }
+
+ @Override
+ public boolean isVolatile() {
+ return true;
+ }
+ }
+
+ /**
+ * Regression test to avoid a potential race condition in {@link ActionExecutionFunction}.
+ *
+ * <p>The test ensures that when ActionExecutionFunction executes a Skyframe-aware action
+ * (implementor of {@link SkyframeAwareAction}), ActionExecutionFunction first requests the inputs
+ * of the action and ensures they are built before requesting any of its Skyframe dependencies.
+ *
+ * <p>This strict ordering is very important to avoid the race condition, which could arise if the
+ * compute method were too eager to request all dependencies: request input files but even if some
+ * are missing, request also the skyframe-dependencies. The race is described in this method's
+ * body.
+ */
+ public void testRaceConditionBetweenInputAcquisitionAndSkyframeDeps() throws Exception {
+ // Sequence of events on threads A and B, showing SkyFunctions and requested SkyKeys, leading
+ // to an InconsistentFilesystemException:
+ //
+ // _______________[Thread A]_________________|_______________[Thread B]_________________
+ // ActionExecutionFunction(gen2_action: | idle
+ // genfiles/gen1 -> genfiles/foo/bar/gen2) |
+ // ARTIFACT:genfiles/gen1 |
+ // MOCK_VALUE:dummy_argument |
+ // env.valuesMissing():yes ==> return |
+ // |
+ // ArtifactFunction(genfiles/gen1) | MockFunction()
+ // CONFIGURED_TARGET://foo:gen1 | FILE:genfiles/foo
+ // ACTION_EXECUTION:gen1_action | env.valuesMissing():yes ==> return
+ // env.valuesMissing():yes ==> return |
+ // | FileFunction(genfiles/foo)
+ // ActionExecutionFunction(gen1_action) | FILE:genfiles
+ // ARTIFACT:genfiles/gen0 | env.valuesMissing():yes ==> return
+ // env.valuesMissing():yes ==> return |
+ // | FileFunction(genfiles)
+ // ArtifactFunction(genfiles/gen0) | FILE_STATE:genfiles
+ // CONFIGURED_TARGET://foo:gen0 | env.valuesMissing():yes ==> return
+ // ACTION_EXECUTION:gen0_action |
+ // env.valuesMissing():yes ==> return | FileStateFunction(genfiles)
+ // | stat genfiles
+ // ActionExecutionFunction(gen0_action) | return FileStateValue:non-existent
+ // create output directory: genfiles |
+ // working | FileFunction(genfiles/foo)
+ // | FILE:genfiles
+ // | FILE_STATE:genfiles/foo
+ // | env.valuesMissing():yes ==> return
+ // |
+ // | FileStateFunction(genfiles/foo)
+ // | stat genfiles/foo
+ // | return FileStateValue:non-existent
+ // |
+ // done, created genfiles/gen0 | FileFunction(genfiles/foo)
+ // return ActionExecutionValue(gen0_action) | FILE:genfiles
+ // | FILE_STATE:genfiles/foo
+ // ArtifactFunction(genfiles/gen0) | return FileValue(genfiles/foo:non-existent)
+ // CONFIGURED_TARGET://foo:gen0 |
+ // ACTION_EXECUTION:gen0_action | MockFunction()
+ // return ArtifactValue(genfiles/gen0) | FILE:genfiles/foo
+ // | FILE:genfiles/foo/bar/gen1
+ // ActionExecutionFunction(gen1_action) | env.valuesMissing():yes ==> return
+ // ARTIFACT:genfiles/gen0 |
+ // create output directory: genfiles/foo/bar | FileFunction(genfiles/foo/bar/gen1)
+ // done, created genfiles/foo/bar/gen1 | FILE:genfiles/foo/bar
+ // return ActionExecutionValue(gen1_action) | env.valuesMissing():yes ==> return
+ // |
+ // idle | FileFunction(genfiles/foo/bar)
+ // | FILE:genfiles/foo
+ // | FILE_STATE:genfiles/foo/bar
+ // | env.valuesMissing():yes ==> return
+ // |
+ // | FileStateFunction(genfiles/foo/bar)
+ // | stat genfiles/foo/bar
+ // | return FileStateValue:directory
+ // |
+ // | FileFunction(genfiles/foo/bar)
+ // | FILE:genfiles/foo
+ // | FILE_STATE:genfiles/foo/bar
+ // | throw InconsistentFilesystemException:
+ // | genfiles/foo doesn't exist but
+ // | genfiles/foo/bar does!
+
+ Artifact genFile1 = createDerivedArtifact("foo/bar/gen1.txt");
+ Artifact genFile2 = createDerivedArtifact("gen2.txt");
+
+ registerAction(
+ new SingleOutputAction(null, genFile1) {
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ writeOutput(null, "gen1");
+ }
+ });
+
+ registerAction(
+ new SingleOutputSkyframeAwareAction(genFile1, genFile2) {
+ @Override
+ public void establishSkyframeDependencies(Environment env) throws ExceptionBase {
+ assertThat(env.valuesMissing()).isFalse();
+ }
+
+ @Override
+ public void execute(ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ writeOutput(readInput(), "gen2");
+ }
+ });
+
+ builder.buildArtifacts(
+ reporter, ImmutableSet.of(genFile2), null, null, null, null, executor, null, false, null);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/SkylarkFileContentHashTests.java b/src/test/java/com/google/devtools/build/lib/skyframe/SkylarkFileContentHashTests.java
new file mode 100644
index 0000000..29470dd
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/SkylarkFileContentHashTests.java
@@ -0,0 +1,165 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.packages.ConstantRuleVisibility;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.skyframe.util.SkyframeExecutorTestUtils;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.util.Collection;
+import java.util.UUID;
+
+/**
+ * Tests for the hash code calculated for Skylark RuleClasses based on the transitive closure
+ * of the imports of their respective definition SkylarkEnvironments.
+ */
+public class SkylarkFileContentHashTests extends BuildViewTestCase {
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ scratch.file("foo/BUILD");
+ scratch.file("bar/BUILD");
+ scratch.file("helper/BUILD");
+
+ scratch.file("helper/ext.bzl", "def rule_impl(ctx):", " return None");
+
+ scratch.file(
+ "foo/ext.bzl",
+ "load('/helper/ext', 'rule_impl')",
+ "",
+ "foo1 = rule(implementation = rule_impl)",
+ "foo2 = rule(implementation = rule_impl)");
+
+ scratch.file(
+ "bar/ext.bzl",
+ "load('/helper/ext', 'rule_impl')",
+ "",
+ "bar1 = rule(implementation = rule_impl)");
+
+ scratch.file(
+ "pkg/BUILD",
+ "load('/foo/ext', 'foo1')",
+ "load('/foo/ext', 'foo2')",
+ "load('/bar/ext', 'bar1')",
+ "",
+ "foo1(name = 'foo1')",
+ "foo2(name = 'foo2')",
+ "bar1(name = 'bar1')");
+ }
+
+ public void testHashInvariance() throws Exception {
+ assertEquals(getHash("pkg", "foo1"), getHash("pkg", "foo1"));
+ }
+
+ public void testHashInvarianceAfterOverwritingFileWithSameContents() throws Exception {
+ String bar1 = getHash("pkg", "bar1");
+ scratch.overwriteFile(
+ "bar/ext.bzl",
+ "load('/helper/ext', 'rule_impl')",
+ "",
+ "bar1 = rule(implementation = rule_impl)");
+ invalidatePackages();
+ assertEquals(bar1, getHash("pkg", "bar1"));
+ }
+
+ public void testHashSameForRulesDefinedInSameFile() throws Exception {
+ assertEquals(getHash("pkg", "foo1"), getHash("pkg", "foo2"));
+ }
+
+ public void testHashNotSameForRulesDefinedInDifferentFiles() throws Exception {
+ assertNotEquals(getHash("pkg", "foo1"), getHash("pkg", "bar1"));
+ }
+
+ public void testImmediateFileChangeChangesHash() throws Exception {
+ String bar1 = getHash("pkg", "bar1");
+ scratch.overwriteFile(
+ "bar/ext.bzl",
+ "load('/helper/ext', 'rule_impl')",
+ "# Some comments to change file hash",
+ "",
+ "bar1 = rule(implementation = rule_impl)");
+ invalidatePackages();
+ assertNotEquals(bar1, getHash("pkg", "bar1"));
+ }
+
+ public void testTransitiveFileChangeChangesHash() throws Exception {
+ String bar1 = getHash("pkg", "bar1");
+ String foo1 = getHash("pkg", "foo1");
+ String foo2 = getHash("pkg", "foo2");
+ scratch.overwriteFile(
+ "helper/ext.bzl",
+ "# Some comments to change file hash",
+ "def rule_impl(ctx):",
+ " return None");
+ invalidatePackages();
+ assertNotEquals(bar1, getHash("pkg", "bar1"));
+ assertNotEquals(foo1, getHash("pkg", "foo1"));
+ assertNotEquals(foo2, getHash("pkg", "foo2"));
+ }
+
+ public void testFileChangeDoesNotAffectRulesDefinedOutsideOfTransitiveClosure() throws Exception {
+ String foo1 = getHash("pkg", "foo1");
+ String foo2 = getHash("pkg", "foo2");
+ scratch.overwriteFile(
+ "bar/ext.bzl",
+ "load('/helper/ext', 'rule_impl')",
+ "# Some comments to change file hash",
+ "",
+ "bar1 = rule(implementation = rule_impl)");
+ invalidatePackages();
+ assertEquals(foo1, getHash("pkg", "foo1"));
+ assertEquals(foo2, getHash("pkg", "foo2"));
+ }
+
+ private void assertNotEquals(String hash, String hash2) {
+ assertFalse(hash.equals(hash2));
+ }
+
+ /**
+ * Returns the hash code of the rule target defined by the pkg and the target name parameters.
+ * Asserts that the targets and it's Skylark dependencies were loaded properly.
+ */
+ private String getHash(String pkg, String name) throws Exception {
+ getSkyframeExecutor()
+ .preparePackageLoading(
+ new PathPackageLocator(rootDirectory),
+ ConstantRuleVisibility.PUBLIC,
+ true,
+ 7,
+ "",
+ UUID.randomUUID());
+ SkyKey pkgLookupKey = PackageValue.key(PackageIdentifier.parse(pkg));
+ EvaluationResult<PackageValue> result =
+ SkyframeExecutorTestUtils.evaluate(
+ getSkyframeExecutor(), pkgLookupKey, /*keepGoing=*/ false, reporter);
+ assertFalse(result.hasError());
+ Collection<Target> targets = result.get(pkgLookupKey).getPackage().getTargets();
+ for (Target target : targets) {
+ if (target.getName().equals(name)) {
+ return ((Rule) target)
+ .getRuleClassObject()
+ .getRuleDefinitionEnvironment()
+ .getTransitiveContentHashCode();
+ }
+ }
+ throw new IllegalStateException("target not found: " + name);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunctionTest.java
new file mode 100644
index 0000000..abd94d9
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/SkylarkImportLookupFunctionTest.java
@@ -0,0 +1,141 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.packages.ConstantRuleVisibility;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.skyframe.util.SkyframeExecutorTestUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.util.UUID;
+
+/**
+ * Tests for SkylarkImportLookupFunction.
+ */
+public class SkylarkImportLookupFunctionTest extends BuildViewTestCase {
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ Path alternativeRoot = scratch.dir("/root_2");
+ getSkyframeExecutor()
+ .preparePackageLoading(
+ new PathPackageLocator(rootDirectory, alternativeRoot),
+ ConstantRuleVisibility.PUBLIC,
+ true,
+ 7,
+ "",
+ UUID.randomUUID());
+ }
+
+ public void testSkylarkImportLabels() throws Exception {
+ scratch.file("pkg1/BUILD");
+ scratch.file("pkg1/ext.bzl");
+ checkLabel("pkg1/ext.bzl", "//pkg1:ext.bzl");
+
+ scratch.file("pkg2/BUILD");
+ scratch.file("pkg2/dir/ext.bzl");
+ checkLabel("pkg2/dir/ext.bzl", "//pkg2:dir/ext.bzl");
+
+ scratch.file("dir/pkg3/BUILD");
+ scratch.file("dir/pkg3/dir/ext.bzl");
+ checkLabel("dir/pkg3/dir/ext.bzl", "//dir/pkg3:dir/ext.bzl");
+ }
+
+ public void testSkylarkImportLabelsAlternativeRoot() throws Exception {
+ scratch.file("/root_2/pkg4/BUILD");
+ scratch.file("/root_2/pkg4/ext.bzl");
+ checkLabel("pkg4/ext.bzl", "//pkg4:ext.bzl");
+ }
+
+ public void testSkylarkImportLabelsMultipleRoot_1() throws Exception {
+ scratch.file("pkg5/BUILD");
+ scratch.file("/root_2/pkg5/ext.bzl");
+ checkLabel("pkg5/ext.bzl", "//pkg5:ext.bzl");
+ }
+
+ public void testSkylarkImportLabelsMultipleRoot_2() throws Exception {
+ scratch.file("/root_2/pkg6/BUILD");
+ scratch.file("pkg6/ext.bzl");
+ checkLabel("pkg6/ext.bzl", "//pkg6:ext.bzl");
+ }
+
+ public void testSkylarkImportLabelsMultipleBuildFiles() throws Exception {
+ scratch.file("dir1/BUILD");
+ scratch.file("dir1/dir2/BUILD");
+ scratch.file("dir1/dir2/ext.bzl");
+ checkLabel("dir1/dir2/ext.bzl", "//dir1/dir2:ext.bzl");
+ }
+
+ public void testLoadRelativePath() throws Exception {
+ scratch.file("pkg/BUILD");
+ scratch.file("pkg/ext1.bzl", "a = 1");
+ scratch.file("pkg/ext2.bzl", "load('ext1', 'a')");
+ get(key("pkg/ext2.bzl"));
+ }
+
+ public void testLoadAbsolutePath() throws Exception {
+ scratch.file("pkg2/BUILD");
+ scratch.file("pkg3/BUILD");
+ scratch.file("pkg2/ext.bzl", "b = 1");
+ scratch.file("pkg3/ext.bzl", "load('/pkg2/ext', 'b')");
+ get(key("pkg3/ext.bzl"));
+ }
+
+ private EvaluationResult<SkylarkImportLookupValue> get(SkyKey skylarkImportLookupKey)
+ throws Exception {
+ EvaluationResult<SkylarkImportLookupValue> result =
+ SkyframeExecutorTestUtils.evaluate(
+ getSkyframeExecutor(), skylarkImportLookupKey, /*keepGoing=*/ false, reporter);
+ if (result.hasError()) {
+ fail(result.getError(skylarkImportLookupKey).getException().getMessage());
+ }
+ return result;
+ }
+
+ private SkyKey key(String file) throws Exception {
+ return SkylarkImportLookupValue.key(
+ PackageIdentifier.createInDefaultRepo(new PathFragment(file)));
+ }
+
+ private void checkLabel(String file, String label) throws Exception {
+ SkyKey skylarkImportLookupKey = key(file);
+ EvaluationResult<SkylarkImportLookupValue> result = get(skylarkImportLookupKey);
+ assertEquals(label, result.get(skylarkImportLookupKey).getDependency().getLabel().toString());
+ }
+
+ public void testSkylarkImportLookupNoBuildFile() throws Exception {
+ scratch.file("pkg/ext.bzl", "");
+ SkyKey skylarkImportLookupKey =
+ SkylarkImportLookupValue.key(
+ PackageIdentifier.createInDefaultRepo(new PathFragment("pkg/ext.bzl")));
+ EvaluationResult<SkylarkImportLookupValue> result =
+ SkyframeExecutorTestUtils.evaluate(
+ getSkyframeExecutor(), skylarkImportLookupKey, /*keepGoing=*/ false, reporter);
+ assertTrue(result.hasError());
+ ErrorInfo errorInfo = result.getError(skylarkImportLookupKey);
+ String errorMessage = errorInfo.getException().getMessage();
+ assertEquals(
+ "Every .bzl file must have a corresponding package, but 'pkg/ext.bzl' "
+ + "does not have one. Please create a BUILD file in the same or any parent directory. "
+ + "Note that this BUILD file does not need to do anything except exist.",
+ errorMessage);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunctionTest.java
new file mode 100644
index 0000000..8ac667f
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/TargetMarkerFunctionTest.java
@@ -0,0 +1,154 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.skyframe.util.SkyframeExecutorTestUtils;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.SkyKey;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Tests for {@link TargetMarkerFunction}. Unfortunately, we can't directly test
+ * TargetMarkerFunction as it uses PackageValues, and PackageFunction uses legacy stuff
+ * that isn't easily mockable. So our testing strategy is to make hacky calls to SkyframeExecutor.
+ */
+public class TargetMarkerFunctionTest extends BuildViewTestCase {
+
+ private SkyframeExecutor skyframeExecutor;
+ private CustomInMemoryFs fs = new CustomInMemoryFs();
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ skyframeExecutor = getSkyframeExecutor();
+ }
+
+ @Override
+ protected FileSystem createFileSystem() {
+ return fs;
+ }
+
+ private SkyKey skyKey(String labelName) throws Exception {
+ return TargetMarkerValue.key(Label.parseAbsolute(labelName));
+ }
+
+ private Throwable getErrorFromTargetValue(String labelName) throws Exception {
+ reporter.removeHandler(failFastHandler);
+ SkyKey targetKey = TargetMarkerValue.key(Label.parseAbsolute(labelName));
+ EvaluationResult<TargetMarkerValue> evaluationResult =
+ SkyframeExecutorTestUtils.evaluate(
+ skyframeExecutor, targetKey, /*keepGoing=*/ false, reporter);
+ Preconditions.checkState(evaluationResult.hasError());
+ reporter.addHandler(failFastHandler);
+ ErrorInfo errorInfo = evaluationResult.getError(skyKey(labelName));
+ // Ensures that TargetFunction rethrows all transitive exceptions.
+ assertEquals(targetKey, Iterables.getOnlyElement(errorInfo.getRootCauses()));
+ return errorInfo.getException();
+ }
+
+ /** Regression test for b/12545745 */
+ public void testLabelCrossingSubpackageBoundary() throws Exception {
+ scratch.file("a/b/c/foo.sh", "echo 'FOO'");
+ scratch.file("a/BUILD", "sh_library(name = 'foo', srcs = ['b/c/foo.sh'])");
+ String labelName = "//a:b/c/foo.sh";
+
+ scratch.file("a/b/BUILD");
+ ModifiedFileSet subpackageBuildFile =
+ ModifiedFileSet.builder().modify(new PathFragment("a/b/BUILD")).build();
+ skyframeExecutor.invalidateFilesUnderPathForTesting(
+ reporter, subpackageBuildFile, rootDirectory);
+
+ NoSuchTargetException exn = (NoSuchTargetException) getErrorFromTargetValue(labelName);
+ // In the presence of b/12545745, the error message is different and comes from the
+ // PackageFunction.
+ assertThat(exn.getMessage())
+ .contains("Label '//a:b/c/foo.sh' crosses boundary of subpackage 'a/b'");
+ }
+
+ public void testNoBuildFileForTargetWithSlash() throws Exception {
+ String labelName = "//no/such/package:target/withslash";
+ BuildFileNotFoundException exn =
+ (BuildFileNotFoundException) getErrorFromTargetValue(labelName);
+ assertEquals(PackageIdentifier.createInDefaultRepo("no/such/package"), exn.getPackageId());
+ String expectedMessage =
+ "no such package 'no/such/package': BUILD file not found on "
+ + "package path for 'no/such/package'";
+ assertThat(exn).hasMessage(expectedMessage);
+ }
+
+ public void testRuleWithError() throws Exception {
+ reporter.removeHandler(failFastHandler);
+ scratch.file(
+ "a/BUILD",
+ "genrule(name = 'conflict1', cmd = '', srcs = [], outs = ['conflict'])",
+ "genrule(name = 'conflict2', cmd = '', srcs = [], outs = ['conflict'])");
+ String labelName = "//a:conflict1";
+ NoSuchTargetException exn = (NoSuchTargetException) getErrorFromTargetValue(labelName);
+ assertThat(exn.getMessage())
+ .contains("Target '//a:conflict1' contains an error and its package is in error");
+ assertEquals(labelName, exn.getLabel().toString());
+ assertTrue(exn.hasTarget());
+ }
+
+ public void testTargetFunctionRethrowsExceptions() throws Exception {
+ reporter.removeHandler(failFastHandler);
+ scratch.file("a/BUILD", "sh_library(name = 'b/c')");
+ Path subpackageBuildFile = scratch.file("a/b/BUILD", "sh_library(name = 'c')");
+ fs.stubStatIOException(subpackageBuildFile, new IOException("nope"));
+ BuildFileNotFoundException exn =
+ (BuildFileNotFoundException) getErrorFromTargetValue("//a:b/c");
+ assertThat(exn.getMessage()).contains("nope");
+ }
+
+ private static class CustomInMemoryFs extends InMemoryFileSystem {
+
+ private Map<Path, IOException> stubbedStatExceptions = Maps.newHashMap();
+
+ public CustomInMemoryFs() {
+ super(BlazeClock.instance());
+ }
+
+ public void stubStatIOException(Path path, IOException stubbedResult) {
+ stubbedStatExceptions.put(path, stubbedResult);
+ }
+
+ @Override
+ public FileStatus stat(Path path, boolean followSymlinks) throws IOException {
+ if (stubbedStatExceptions.containsKey(path)) {
+ throw stubbedStatExceptions.get(path);
+ }
+ return super.stat(path, followSymlinks);
+ }
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderMediumTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderMediumTest.java
new file mode 100644
index 0000000..7ecc956
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderMediumTest.java
@@ -0,0 +1,472 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.cache.CompactPersistentActionCache;
+import com.google.devtools.build.lib.testutil.BlazeTestUtils;
+import com.google.devtools.build.lib.testutil.Suite;
+import com.google.devtools.build.lib.testutil.TestSpec;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+
+import java.io.IOException;
+
+/**
+ * These tests belong to {@link TimestampBuilderTest}, but they're in a
+ * separate class for now because they are a little slower.
+ */
+@TestSpec(size = Suite.MEDIUM_TESTS)
+public class TimestampBuilderMediumTest extends TimestampBuilderTestCase {
+ private Path cacheRoot;
+ private CompactPersistentActionCache cache;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ // BlazeRuntime.setupLogging(Level.FINEST); // Uncomment this for debugging.
+
+ cacheRoot = scratch.dir("cacheRoot");
+ cache = createCache();
+ }
+
+ private CompactPersistentActionCache createCache() throws IOException {
+ return new CompactPersistentActionCache(cacheRoot, clock);
+ }
+
+ /**
+ * Creates and returns a new caching builder based on a given {@code cache}.
+ */
+ private Builder persistentBuilder(CompactPersistentActionCache cache) {
+ return createBuilder(cache);
+ }
+
+ // TODO(blaze-team): (2009) :
+ // - test timestamp monotonicity is not required (i.e. set mtime backwards)
+ // - test change of key causes rebuild
+
+ public void testUnneededInputs() throws Exception {
+ Artifact hello = createSourceArtifact("hello");
+ BlazeTestUtils.makeEmptyFile(hello.getPath());
+ Artifact optional = createSourceArtifact("hello.optional");
+ Artifact goodbye = createDerivedArtifact("goodbye");
+ Button button = createActionButton(Sets.newHashSet(hello, optional), Sets.newHashSet(goodbye));
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // built
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+
+ // Creating a new persistent cache does not cause a rebuild
+ cache.save();
+ cache = createCache();
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertFalse(button.pressed); // not rebuilt
+
+ BlazeTestUtils.makeEmptyFile(optional.getPath());
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // built
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+
+ optional.getPath().delete();
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // built
+
+ // Creating a new persistent cache does not cause a rebuild
+ cache.save();
+ cache = createCache();
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertFalse(button.pressed); // not rebuilt
+ }
+
+ public void testPersistentCache_ModifyingInputCausesActionReexecution() throws Exception {
+ // /hello -> [action] -> /goodbye
+ Artifact hello = createSourceArtifact("hello");
+ BlazeTestUtils.makeEmptyFile(hello.getPath());
+ Artifact goodbye = createDerivedArtifact("goodbye");
+ Button button = createActionButton(Sets.newHashSet(hello), Sets.newHashSet(goodbye));
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // built
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+
+ FileSystemUtils.touchFile(hello.getPath());
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // rebuilt
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+
+ // Creating a new persistent cache does not cause a rebuild
+ cache.save();
+ buildArtifacts(persistentBuilder(createCache()), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+ }
+
+ public void testModifyingInputCausesActionReexecution() throws Exception {
+ // /hello -> [action] -> /goodbye
+ Artifact hello = createSourceArtifact("hello");
+ FileSystemUtils.createDirectoryAndParents(hello.getPath().getParentDirectory());
+ FileSystemUtils.writeContentAsLatin1(hello.getPath(), "content1");
+ Artifact goodbye = createDerivedArtifact("goodbye");
+ Button button = createActionButton(Sets.newHashSet(hello), Sets.newHashSet(goodbye));
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // built
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // still not rebuilt
+
+ FileSystemUtils.writeContentAsLatin1(hello.getPath(), "content2");
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // rebuilt
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+
+ // Creating a new persistent cache does not cause a rebuild
+ cache.save();
+ buildArtifacts(persistentBuilder(createCache()), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+ }
+
+ public void testArtifactOrderingDoesNotMatter() throws Exception {
+ // (/hello,/there) -> [action] -> /goodbye
+
+ Artifact hello = createSourceArtifact("hello");
+ Artifact there = createSourceArtifact("there");
+ FileSystemUtils.createDirectoryAndParents(hello.getPath().getParentDirectory());
+ FileSystemUtils.writeContentAsLatin1(hello.getPath(), "hello");
+ FileSystemUtils.writeContentAsLatin1(there.getPath(), "there");
+ Artifact goodbye = createDerivedArtifact("goodbye");
+ Button button =
+ createActionButton(
+ Sets.newLinkedHashSet(ImmutableList.of(hello, there)), Sets.newHashSet(goodbye));
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // built
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+
+ // Now create duplicate graph, with swapped order.
+ clearActions();
+ Artifact goodbye2 = createDerivedArtifact("goodbye");
+ Button button2 =
+ createActionButton(
+ Sets.newLinkedHashSet(ImmutableList.of(there, hello)), Sets.newHashSet(goodbye2));
+
+ button2.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button2.pressed); // still not rebuilt
+ }
+
+ public void testOldCacheKeysAreCleanedUp() throws Exception {
+ // [action1] -> (/goodbye), cache key will be /goodbye
+ Artifact goodbye = createDerivedArtifact("goodbye");
+ FileSystemUtils.createDirectoryAndParents(goodbye.getPath().getParentDirectory());
+ FileSystemUtils.writeContentAsLatin1(goodbye.getPath(), "test");
+ Button button = createActionButton(emptySet, Sets.newLinkedHashSet(ImmutableList.of(goodbye)));
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // built
+
+ // action1 is cached using the cache key /goodbye.
+ assertThat(cache.get(goodbye.getExecPathString())).isNotNull();
+
+ // [action2] -> (/hello,/goodbye), cache key will be /hello
+ clearActions();
+ Artifact hello = createDerivedArtifact("hello");
+ Artifact goodbye2 = createDerivedArtifact("goodbye");
+ Button button2 =
+ createActionButton(emptySet, Sets.newLinkedHashSet(ImmutableList.of(hello, goodbye2)));
+
+ button2.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello, goodbye2);
+ assertTrue(button2.pressed); // rebuilt
+
+ // action2 is cached using the cache key /hello.
+ assertThat(cache.get(hello.getExecPathString())).isNotNull();
+
+ // Now, action1 should no longer be in the cache.
+ assertThat(cache.get(goodbye.getExecPathString())).isNull();
+ }
+
+ public void testArtifactNamesMatter() throws Exception {
+ // /hello -> [action] -> /goodbye
+
+ Artifact hello = createSourceArtifact("hello");
+ FileSystemUtils.createDirectoryAndParents(hello.getPath().getParentDirectory());
+ FileSystemUtils.writeContentAsLatin1(hello.getPath(), "hello");
+ Artifact goodbye = createDerivedArtifact("goodbye");
+ Button button = createActionButton(Sets.newHashSet(hello), Sets.newHashSet(goodbye));
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // built
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+
+ // Now create duplicate graph, replacing "hello" with "hi".
+ clearActions();
+ Artifact hi = createSourceArtifact("hi");
+ FileSystemUtils.createDirectoryAndParents(hi.getPath().getParentDirectory());
+ FileSystemUtils.writeContentAsLatin1(hi.getPath(), "hello");
+ Artifact goodbye2 = createDerivedArtifact("goodbye");
+ Button button2 = createActionButton(Sets.newHashSet(hi), Sets.newHashSet(goodbye2));
+
+ button2.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye2);
+ assertTrue(button2.pressed); // name changed. must rebuild.
+ }
+
+ public void testDuplicateInputs() throws Exception {
+ // (/hello,/hello) -> [action] -> /goodbye
+
+ Artifact hello = createSourceArtifact("hello");
+ FileSystemUtils.createDirectoryAndParents(hello.getPath().getParentDirectory());
+ FileSystemUtils.writeContentAsLatin1(hello.getPath(), "hello");
+ Artifact goodbye = createDerivedArtifact("goodbye");
+ Button button =
+ createActionButton(Lists.<Artifact>newArrayList(hello, hello), Sets.newHashSet(goodbye));
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // built
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+
+ FileSystemUtils.writeContentAsLatin1(hello.getPath(), "hello2");
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // rebuilt
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+
+ // Creating a new persistent cache does not cause a rebuild
+ cache.save();
+ buildArtifacts(persistentBuilder(createCache()), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+ }
+
+ /**
+ * Tests that changing timestamp of the input file without changing it content
+ * does not cause action reexecution when metadata cache uses file digests in
+ * addition to the timestamp.
+ */
+ public void testModifyingTimestampOnlyDoesNotCauseActionReexecution() throws Exception {
+ // /hello -> [action] -> /goodbye
+ Artifact hello = createSourceArtifact("hello");
+ FileSystemUtils.createDirectoryAndParents(hello.getPath().getParentDirectory());
+ FileSystemUtils.writeContentAsLatin1(hello.getPath(), "content1");
+ Artifact goodbye = createDerivedArtifact("goodbye");
+ Button button = createActionButton(Sets.newHashSet(hello), Sets.newHashSet(goodbye));
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertTrue(button.pressed); // built
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), goodbye);
+ assertFalse(button.pressed); // not rebuilt
+
+ // Creating a new persistent caches, including metadata cache does not cause
+ // a rebuild
+ cache.save();
+ Builder builder = persistentBuilder(createCache());
+ buildArtifacts(builder, goodbye);
+ assertFalse(button.pressed); // not rebuilt
+ }
+
+ public void testPersistentCache_ModifyingOutputCausesActionReexecution() throws Exception {
+ // [action] -> /hello
+ Artifact hello = createDerivedArtifact("hello");
+ Button button = createActionButton(emptySet, Sets.newHashSet(hello));
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertTrue(button.pressed); // built
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertFalse(button.pressed); // not rebuilt
+
+ BlazeTestUtils.changeModtime(hello.getPath());
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertTrue(button.pressed); // rebuilt
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertFalse(button.pressed); // not rebuilt
+
+ // Creating a new persistent cache does not cause a rebuild
+ cache.save();
+ buildArtifacts(persistentBuilder(createCache()), hello);
+ assertFalse(button.pressed); // not rebuilt
+ }
+
+ public void testPersistentCache_missingFilenameIndexCausesActionReexecution() throws Exception {
+ // [action] -> /hello
+ Artifact hello = createDerivedArtifact("hello");
+ Button button = createActionButton(emptySet, Sets.newHashSet(hello));
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertTrue(button.pressed); // built
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertFalse(button.pressed); // not rebuilt
+
+ BlazeTestUtils.changeModtime(hello.getPath());
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertTrue(button.pressed); // rebuilt
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertFalse(button.pressed); // not rebuilt
+
+ // Creating a new persistent cache does not cause a rebuild
+ cache.save();
+
+ // Remove filename index file.
+ assertTrue(
+ Iterables.getOnlyElement(
+ UnixGlob.forPath(cacheRoot).addPattern("filename_index*").globInterruptible())
+ .delete());
+
+ // Now first cache creation attempt should cause IOException while renaming corrupted files.
+ // Second attempt will initialize empty cache, causing rebuild.
+ try {
+ createCache();
+ fail("Expected IOException");
+ } catch (IOException e) {
+ assertThat(e).hasMessage("Failed action cache referential integrity check: empty index");
+ }
+
+ buildArtifacts(persistentBuilder(createCache()), hello);
+ assertTrue(button.pressed); // rebuilt due to the missing filename index
+ }
+
+ public void testPersistentCache_failedIntegrityCheckCausesActionReexecution() throws Exception {
+ // [action] -> /hello
+ Artifact hello = createDerivedArtifact("hello");
+ Button button = createActionButton(emptySet, Sets.newHashSet(hello));
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertTrue(button.pressed); // built
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertFalse(button.pressed); // not rebuilt
+
+ BlazeTestUtils.changeModtime(hello.getPath());
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertTrue(button.pressed); // rebuilt
+
+ button.pressed = false;
+ buildArtifacts(persistentBuilder(cache), hello);
+ assertFalse(button.pressed); // not rebuilt
+
+ cache.save();
+
+ // Get filename index path and store a copy of it.
+ Path indexPath =
+ Iterables.getOnlyElement(
+ UnixGlob.forPath(cacheRoot).addPattern("filename_index*").globInterruptible());
+ Path indexCopy = scratch.resolve("index_copy");
+ FileSystemUtils.copyFile(indexPath, indexCopy);
+
+ // Add extra records to the action cache and indexer.
+ Artifact helloExtra = createDerivedArtifact("hello_extra");
+ Button buttonExtra = createActionButton(emptySet, Sets.newHashSet(helloExtra));
+ buildArtifacts(persistentBuilder(cache), helloExtra);
+ assertTrue(buttonExtra.pressed); // built
+
+ cache.save();
+ assertTrue(indexPath.getFileSize() > indexCopy.getFileSize());
+
+ // Validate current cache.
+ buildArtifacts(persistentBuilder(createCache()), hello);
+ assertFalse(button.pressed); // not rebuilt
+
+ // Restore outdated file index.
+ FileSystemUtils.copyFile(indexCopy, indexPath);
+
+ // Now first cache creation attempt should cause IOException while renaming corrupted files.
+ // Second attempt will initialize empty cache, causing rebuild.
+ try {
+ createCache();
+ fail("Expected IOException");
+ } catch (IOException e) {
+ assertThat(e.getMessage()).contains("Failed action cache referential integrity check");
+ }
+
+ // Validate cache with incorrect (out-of-date) filename index.
+ buildArtifacts(persistentBuilder(createCache()), hello);
+ assertTrue(button.pressed); // rebuilt due to the out-of-date index
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java
new file mode 100644
index 0000000..abd081c
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java
@@ -0,0 +1,385 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.devtools.build.lib.actions.util.ActionCacheTestHelper.AMNESIAC_CACHE;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Range;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionCacheChecker;
+import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
+import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.BuildFailedException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.actions.TestExecException;
+import com.google.devtools.build.lib.actions.cache.ActionCache;
+import com.google.devtools.build.lib.actions.util.DummyExecutor;
+import com.google.devtools.build.lib.actions.util.TestAction;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.buildtool.SkyframeBuilder;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.skyframe.ActionExecutionFunction;
+import com.google.devtools.build.lib.skyframe.ActionLookupValue;
+import com.google.devtools.build.lib.skyframe.ArtifactFunction;
+import com.google.devtools.build.lib.skyframe.ArtifactValue;
+import com.google.devtools.build.lib.skyframe.AspectValue;
+import com.google.devtools.build.lib.skyframe.Builder;
+import com.google.devtools.build.lib.skyframe.ExternalFilesHelper;
+import com.google.devtools.build.lib.skyframe.FileFunction;
+import com.google.devtools.build.lib.skyframe.FileStateFunction;
+import com.google.devtools.build.lib.skyframe.PrecomputedValue;
+import com.google.devtools.build.lib.skyframe.SkyFunctions;
+import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor;
+import com.google.devtools.build.lib.testutil.FoundationTestCase;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.CycleInfo;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
+import com.google.devtools.build.skyframe.RecordingDifferencer;
+import com.google.devtools.build.skyframe.SequentialBuildDriver;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * The common code that's shared between various builder tests.
+ */
+public abstract class TimestampBuilderTestCase extends FoundationTestCase {
+
+ private static final SkyKey OWNER_KEY = new SkyKey(SkyFunctions.ACTION_LOOKUP, "OWNER");
+ protected static final ActionLookupValue.ActionLookupKey ALL_OWNER =
+ new SingletonActionLookupKey();
+ protected static final Predicate<Action> ALWAYS_EXECUTE_FILTER = Predicates.alwaysTrue();
+ protected static final String CYCLE_MSG = "Yarrrr, there be a cycle up in here";
+
+ protected Clock clock = BlazeClock.instance();
+ protected TimestampGranularityMonitor tsgm;
+ protected RecordingDifferencer differencer = new RecordingDifferencer();
+ private Set<Action> actions;
+
+ protected AtomicReference<EventBus> eventBusRef = new AtomicReference<>();
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ inMemoryCache = new InMemoryActionCache();
+ tsgm = new TimestampGranularityMonitor(clock);
+ ResourceManager.instance().setAvailableResources(ResourceSet.createWithRamCpuIo(100, 1, 1));
+ actions = new HashSet<>();
+ }
+
+ protected void clearActions() {
+ actions.clear();
+ }
+
+ protected <T extends Action> T registerAction(T action) {
+ actions.add(action);
+ return action;
+ }
+
+ protected Builder createBuilder(ActionCache actionCache) {
+ return createBuilder(actionCache, 1, /*keepGoing=*/ false);
+ }
+
+ /**
+ * Create a ParallelBuilder with a DatabaseDependencyChecker using the
+ * specified ActionCache.
+ */
+ protected Builder createBuilder(
+ final ActionCache actionCache, final int threadCount, final boolean keepGoing) {
+ return createBuilder(actionCache, threadCount, keepGoing, null);
+ }
+
+ protected Builder createBuilder(
+ final ActionCache actionCache,
+ final int threadCount,
+ final boolean keepGoing,
+ @Nullable EvaluationProgressReceiver evaluationProgressReceiver) {
+ AtomicReference<PathPackageLocator> pkgLocator =
+ new AtomicReference<>(new PathPackageLocator());
+ ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator);
+ differencer = new RecordingDifferencer();
+
+ ActionExecutionStatusReporter statusReporter =
+ ActionExecutionStatusReporter.create(new StoredEventHandler());
+ final SkyframeActionExecutor skyframeActionExecutor =
+ new SkyframeActionExecutor(
+ ResourceManager.instance(), eventBusRef, new AtomicReference<>(statusReporter));
+
+ skyframeActionExecutor.setActionLogBufferPathGenerator(
+ new ActionLogBufferPathGenerator(actionOutputBase));
+
+ final InMemoryMemoizingEvaluator evaluator =
+ new InMemoryMemoizingEvaluator(
+ ImmutableMap.of(
+ SkyFunctions.FILE_STATE,
+ new FileStateFunction(tsgm, externalFilesHelper),
+ SkyFunctions.FILE,
+ new FileFunction(pkgLocator, tsgm, externalFilesHelper),
+ SkyFunctions.ARTIFACT,
+ new ArtifactFunction(Predicates.<PathFragment>alwaysFalse()),
+ SkyFunctions.ACTION_EXECUTION,
+ new ActionExecutionFunction(skyframeActionExecutor, tsgm)),
+ differencer,
+ evaluationProgressReceiver);
+ final SequentialBuildDriver driver = new SequentialBuildDriver(evaluator);
+ PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
+
+ return new Builder() {
+ private void setGeneratingActions() {
+ if (evaluator.getExistingValueForTesting(OWNER_KEY) == null) {
+ differencer.inject(ImmutableMap.of(OWNER_KEY, new ActionLookupValue(actions)));
+ }
+ }
+
+ @Override
+ public void buildArtifacts(
+ Reporter reporter,
+ Set<Artifact> artifacts,
+ Set<ConfiguredTarget> parallelTests,
+ Set<ConfiguredTarget> exclusiveTests,
+ Collection<ConfiguredTarget> targetsToBuild,
+ Collection<AspectValue> aspects,
+ Executor executor,
+ Set<ConfiguredTarget> builtTargets,
+ boolean explain,
+ Range<Long> lastExecutionTimeRange)
+ throws BuildFailedException, AbruptExitException, InterruptedException,
+ TestExecException {
+ skyframeActionExecutor.prepareForExecution(
+ reporter,
+ executor,
+ keepGoing, /*explain=*/
+ false,
+ new ActionCacheChecker(actionCache, null, ALWAYS_EXECUTE_FILTER, false));
+
+ List<SkyKey> keys = new ArrayList<>();
+ for (Artifact artifact : artifacts) {
+ keys.add(ArtifactValue.key(artifact, true));
+ }
+
+ setGeneratingActions();
+ EvaluationResult<SkyValue> result = driver.evaluate(keys, keepGoing, threadCount, reporter);
+
+ if (result.hasError()) {
+ boolean hasCycles = false;
+ for (Map.Entry<SkyKey, ErrorInfo> entry : result.errorMap().entrySet()) {
+ Iterable<CycleInfo> cycles = entry.getValue().getCycleInfo();
+ hasCycles |= !Iterables.isEmpty(cycles);
+ }
+ if (hasCycles) {
+ throw new BuildFailedException(CYCLE_MSG);
+ } else if (result.errorMap().isEmpty() || keepGoing) {
+ throw new BuildFailedException();
+ } else {
+ SkyframeBuilder.rethrow(Preconditions.checkNotNull(result.getError().getException()));
+ }
+ }
+ }
+ };
+ }
+
+ /** A non-persistent cache. */
+ protected InMemoryActionCache inMemoryCache;
+
+ /** A class that records an event. */
+ protected static class Button implements Runnable {
+ protected boolean pressed = false;
+
+ @Override
+ public void run() {
+ pressed = true;
+ }
+ }
+
+ /** A class that counts occurrences of an event. */
+ static class Counter implements Runnable {
+ int count = 0;
+
+ @Override
+ public void run() {
+ count++;
+ }
+ }
+
+ Artifact createSourceArtifact(String name) {
+ return createSourceArtifact(scratch.getFileSystem(), name);
+ }
+
+ Artifact createSourceArtifact(FileSystem fs, String name) {
+ Path root = fs.getPath(TestUtils.tmpDir());
+ return new Artifact(new PathFragment(name), Root.asSourceRoot(root));
+ }
+
+ protected Artifact createDerivedArtifact(String name) {
+ return createDerivedArtifact(scratch.getFileSystem(), name);
+ }
+
+ Artifact createDerivedArtifact(FileSystem fs, String name) {
+ Path execRoot = fs.getPath(TestUtils.tmpDir());
+ PathFragment execPath = new PathFragment("out").getRelative(name);
+ Path path = execRoot.getRelative(execPath);
+ return new Artifact(
+ path, Root.asDerivedRoot(execRoot, execRoot.getRelative("out")), execPath, ALL_OWNER);
+ }
+
+ /**
+ * Creates and returns a new "amnesiac" builder based on the amnesiac cache.
+ */
+ protected Builder amnesiacBuilder() {
+ return createBuilder(AMNESIAC_CACHE);
+ }
+
+ /**
+ * Creates and returns a new caching builder based on the inMemoryCache.
+ */
+ protected Builder cachingBuilder() {
+ return createBuilder(inMemoryCache);
+ }
+
+ /**
+ * Creates a TestAction from 'inputs' to 'outputs', and a new button, such
+ * that executing the action causes the button to be pressed. The button is
+ * returned.
+ */
+ protected Button createActionButton(Collection<Artifact> inputs, Collection<Artifact> outputs) {
+ Button button = new Button();
+ registerAction(new TestAction(button, inputs, outputs));
+ return button;
+ }
+
+ /**
+ * Creates a TestAction from 'inputs' to 'outputs', and a new counter, such
+ * that executing the action causes the counter to be incremented. The
+ * counter is returned.
+ */
+ protected Counter createActionCounter(Collection<Artifact> inputs, Collection<Artifact> outputs) {
+ Counter counter = new Counter();
+ registerAction(new TestAction(counter, inputs, outputs));
+ return counter;
+ }
+
+ protected static Set<Artifact> emptySet = Collections.emptySet();
+
+ protected void buildArtifacts(Builder builder, Artifact... artifacts)
+ throws BuildFailedException, AbruptExitException, InterruptedException, TestExecException {
+
+ tsgm.setCommandStartTime();
+ Set<Artifact> artifactsToBuild = Sets.newHashSet(artifacts);
+ Set<ConfiguredTarget> builtArtifacts = new HashSet<>();
+ try {
+ builder.buildArtifacts(
+ reporter,
+ artifactsToBuild,
+ null,
+ null,
+ null,
+ null,
+ new DummyExecutor(rootDirectory),
+ builtArtifacts, /*explain=*/
+ false,
+ null);
+ } finally {
+ tsgm.waitForTimestampGranularity(reporter.getOutErr());
+ }
+ }
+
+ protected static class InMemoryActionCache implements ActionCache {
+
+ private final Map<String, Entry> actionCache = new HashMap<>();
+
+ @Override
+ public synchronized void put(String key, ActionCache.Entry entry) {
+ actionCache.put(key, entry);
+ }
+
+ @Override
+ public synchronized Entry get(String key) {
+ return actionCache.get(key);
+ }
+
+ @Override
+ public synchronized void remove(String key) {
+ actionCache.remove(key);
+ }
+
+ @Override
+ public Entry createEntry(String key) {
+ return new ActionCache.Entry(key);
+ }
+
+ public synchronized void reset() {
+ actionCache.clear();
+ }
+
+ @Override
+ public long save() {
+ // safe to ignore
+ return 0;
+ }
+
+ @Override
+ public void dump(PrintStream out) {
+ out.println("In-memory action cache has " + actionCache.size() + " records");
+ }
+ }
+
+ private static class SingletonActionLookupKey extends ActionLookupValue.ActionLookupKey {
+ @Override
+ SkyKey getSkyKey() {
+ return OWNER_KEY;
+ }
+
+ @Override
+ SkyFunctionName getType() {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
