Open source some skyframe/bazel tests.

--
MOS_MIGRATED_REVID=106308990
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FilesystemValueCheckerTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/FilesystemValueCheckerTest.java
new file mode 100644
index 0000000..d917c8e
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/FilesystemValueCheckerTest.java
@@ -0,0 +1,526 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.Runnables;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Root;
+import com.google.devtools.build.lib.actions.util.TestAction;
+import com.google.devtools.build.lib.events.NullEventHandler;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.BasicFilesystemDirtinessChecker;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.BatchStat;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
+import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.RootedPath;
+import com.google.devtools.build.lib.vfs.Symlinks;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+import com.google.devtools.build.skyframe.Differencer.Diff;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.RecordingDifferencer;
+import com.google.devtools.build.skyframe.SequentialBuildDriver;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.annotation.Nullable;
+
+/**
+ * Tests for {@link FilesystemValueChecker}.
+ */
+public class FilesystemValueCheckerTest extends TestCase {
+
+  private RecordingDifferencer differencer;
+  private MemoizingEvaluator evaluator;
+  private SequentialBuildDriver driver;
+  private MockFileSystem fs;
+  private Path pkgRoot;
+  private TimestampGranularityMonitor tsgm;
+
+  @Override
+  protected void setUp() throws Exception {
+    super.setUp();
+
+    ImmutableMap.Builder<SkyFunctionName, SkyFunction> skyFunctions = ImmutableMap.builder();
+
+    fs = new MockFileSystem();
+    pkgRoot = fs.getPath("/testroot");
+
+    tsgm = new TimestampGranularityMonitor(BlazeClock.instance());
+    AtomicReference<PathPackageLocator> pkgLocator =
+        new AtomicReference<>(new PathPackageLocator());
+    ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator);
+    skyFunctions.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
+    skyFunctions.put(SkyFunctions.FILE, new FileFunction(pkgLocator, tsgm, externalFilesHelper));
+    skyFunctions.put(
+        SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS, new FileSymlinkCycleUniquenessFunction());
+    skyFunctions.put(
+        SkyFunctions.FILE_SYMLINK_INFINITE_EXPANSION_UNIQUENESS,
+        new FileSymlinkInfiniteExpansionUniquenessFunction());
+    differencer = new RecordingDifferencer();
+    evaluator = new InMemoryMemoizingEvaluator(skyFunctions.build(), differencer);
+    driver = new SequentialBuildDriver(evaluator);
+    PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
+  }
+
+  @Override
+  protected void tearDown() throws Exception {
+    super.tearDown();
+  }
+
+  public void testEmpty() throws Exception {
+    FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+    assertEmptyDiff(getDirtyFilesystemKeys(checker));
+  }
+
+  public void testSimple() throws Exception {
+    FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+
+    Path path = fs.getPath("/foo");
+    FileSystemUtils.createEmptyFile(path);
+    assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+    SkyKey skyKey =
+        FileStateValue.key(RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("foo")));
+    EvaluationResult<SkyValue> result =
+        driver.evaluate(
+            ImmutableList.of(skyKey),
+            false,
+            SkyframeExecutor.DEFAULT_THREAD_COUNT,
+            NullEventHandler.INSTANCE);
+    assertFalse(result.hasError());
+
+    assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+    FileSystemUtils.writeContentAsLatin1(path, "hello");
+    assertDiffWithNewValues(getDirtyFilesystemKeys(checker), skyKey);
+
+    // The dirty bits are not reset until the FileValues are actually revalidated.
+    assertDiffWithNewValues(getDirtyFilesystemKeys(checker), skyKey);
+
+    differencer.invalidate(ImmutableList.of(skyKey));
+    result =
+        driver.evaluate(
+            ImmutableList.of(skyKey),
+            false,
+            SkyframeExecutor.DEFAULT_THREAD_COUNT,
+            NullEventHandler.INSTANCE);
+    assertFalse(result.hasError());
+    assertEmptyDiff(getDirtyFilesystemKeys(checker));
+  }
+
+  /**
+   * Tests that an already-invalidated value can still be marked changed: symlink points at sym1.
+   * Invalidate symlink by changing sym1 from pointing at path to point to sym2. This only dirties
+   * (rather than changes) symlink because sym2 still points at path, so all symlink stats remain
+   * the same. Then do a null build, change sym1 back to point at path, and change symlink to not be
+   * a symlink anymore. The fact that it is not a symlink should be detected.
+   */
+  public void testDirtySymlink() throws Exception {
+    FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+
+    Path path = fs.getPath("/foo");
+    FileSystemUtils.writeContentAsLatin1(path, "foo contents");
+    // We need the intermediate sym1 and sym2 so that we can dirty a child of symlink without
+    // actually changing the FileValue calculated for symlink (if we changed the contents of foo,
+    // the the FileValue created for symlink would notice, since it stats foo).
+    Path sym1 = fs.getPath("/sym1");
+    Path sym2 = fs.getPath("/sym2");
+    Path symlink = fs.getPath("/bar");
+    FileSystemUtils.ensureSymbolicLink(symlink, sym1);
+    FileSystemUtils.ensureSymbolicLink(sym1, path);
+    FileSystemUtils.ensureSymbolicLink(sym2, path);
+    SkyKey fooKey =
+        FileValue.key(RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("foo")));
+    RootedPath symlinkRootedPath =
+        RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("bar"));
+    SkyKey symlinkKey = FileValue.key(symlinkRootedPath);
+    SkyKey symlinkFileStateKey = FileStateValue.key(symlinkRootedPath);
+    RootedPath sym1RootedPath =
+        RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("sym1"));
+    SkyKey sym1FileStateKey = FileStateValue.key(sym1RootedPath);
+    Iterable<SkyKey> allKeys = ImmutableList.of(symlinkKey, fooKey);
+
+    // First build -- prime the graph.
+    EvaluationResult<FileValue> result =
+        driver.evaluate(
+            allKeys, false, SkyframeExecutor.DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
+    assertFalse(result.hasError());
+    FileValue symlinkValue = result.get(symlinkKey);
+    FileValue fooValue = result.get(fooKey);
+    assertTrue(symlinkValue.toString(), symlinkValue.isSymlink());
+    // Digest is not always available, so use size as a proxy for contents.
+    assertEquals(fooValue.getSize(), symlinkValue.getSize());
+    assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+    // Before second build, move sym1 to point to sym2.
+    assertTrue(sym1.delete());
+    FileSystemUtils.ensureSymbolicLink(sym1, sym2);
+    assertDiffWithNewValues(getDirtyFilesystemKeys(checker), sym1FileStateKey);
+
+    differencer.invalidate(ImmutableList.of(sym1FileStateKey));
+    result =
+        driver.evaluate(
+            ImmutableList.<SkyKey>of(),
+            false,
+            SkyframeExecutor.DEFAULT_THREAD_COUNT,
+            NullEventHandler.INSTANCE);
+    assertFalse(result.hasError());
+    assertDiffWithNewValues(getDirtyFilesystemKeys(checker), sym1FileStateKey);
+
+    // Before third build, move sym1 back to original (so change pruning will prevent signaling of
+    // its parents, but change symlink for real.
+    assertTrue(sym1.delete());
+    FileSystemUtils.ensureSymbolicLink(sym1, path);
+    assertTrue(symlink.delete());
+    FileSystemUtils.writeContentAsLatin1(symlink, "new symlink contents");
+    assertDiffWithNewValues(getDirtyFilesystemKeys(checker), symlinkFileStateKey);
+    differencer.invalidate(ImmutableList.of(symlinkFileStateKey));
+    result =
+        driver.evaluate(
+            allKeys, false, SkyframeExecutor.DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
+    assertFalse(result.hasError());
+    symlinkValue = result.get(symlinkKey);
+    assertFalse(symlinkValue.toString(), symlinkValue.isSymlink());
+    assertEquals(fooValue, result.get(fooKey));
+    assertThat(symlinkValue.getSize()).isNotEqualTo(fooValue.getSize());
+    assertEmptyDiff(getDirtyFilesystemKeys(checker));
+  }
+
+  public void testExplicitFiles() throws Exception {
+    FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+
+    Path path1 = fs.getPath("/foo1");
+    Path path2 = fs.getPath("/foo2");
+    FileSystemUtils.createEmptyFile(path1);
+    FileSystemUtils.createEmptyFile(path2);
+    assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+    SkyKey key1 =
+        FileStateValue.key(
+            RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("foo1")));
+    SkyKey key2 =
+        FileStateValue.key(
+            RootedPath.toRootedPath(fs.getRootDirectory(), new PathFragment("foo2")));
+    Iterable<SkyKey> skyKeys = ImmutableList.of(key1, key2);
+    EvaluationResult<SkyValue> result =
+        driver.evaluate(
+            skyKeys, false, SkyframeExecutor.DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
+    assertFalse(result.hasError());
+
+    assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+    FileSystemUtils.writeContentAsLatin1(path1, "hello1");
+    FileSystemUtils.writeContentAsLatin1(path1, "hello2");
+    path1.setLastModifiedTime(27);
+    path2.setLastModifiedTime(42);
+    assertDiffWithNewValues(getDirtyFilesystemKeys(checker), key1, key2);
+
+    differencer.invalidate(skyKeys);
+    result =
+        driver.evaluate(
+            skyKeys, false, SkyframeExecutor.DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
+    assertFalse(result.hasError());
+    assertEmptyDiff(getDirtyFilesystemKeys(checker));
+  }
+
+  public void testFileWithIOExceptionNotConsideredDirty() throws Exception {
+    Path path = fs.getPath("/testroot/foo");
+    path.getParentDirectory().createDirectory();
+    path.createSymbolicLink(new PathFragment("bar"));
+
+    fs.readlinkThrowsIoException = true;
+    SkyKey fileKey = FileStateValue.key(RootedPath.toRootedPath(pkgRoot, new PathFragment("foo")));
+    EvaluationResult<SkyValue> result =
+        driver.evaluate(
+            ImmutableList.of(fileKey),
+            false,
+            SkyframeExecutor.DEFAULT_THREAD_COUNT,
+            NullEventHandler.INSTANCE);
+    assertTrue(result.hasError());
+
+    fs.readlinkThrowsIoException = false;
+    FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+    Diff diff = getDirtyFilesystemKeys(checker);
+    assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
+    assertThat(diff.changedKeysWithNewValues()).isEmpty();
+  }
+
+  public void testFilesInCycleNotConsideredDirty() throws Exception {
+    Path path1 = pkgRoot.getRelative("foo1");
+    Path path2 = pkgRoot.getRelative("foo2");
+    Path path3 = pkgRoot.getRelative("foo3");
+    FileSystemUtils.ensureSymbolicLink(path1, path2);
+    FileSystemUtils.ensureSymbolicLink(path2, path3);
+    FileSystemUtils.ensureSymbolicLink(path3, path1);
+    SkyKey fileKey1 = FileValue.key(RootedPath.toRootedPath(pkgRoot, path1));
+
+    EvaluationResult<SkyValue> result =
+        driver.evaluate(
+            ImmutableList.of(fileKey1),
+            false,
+            SkyframeExecutor.DEFAULT_THREAD_COUNT,
+            NullEventHandler.INSTANCE);
+    assertTrue(result.hasError());
+
+    FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+    Diff diff = getDirtyFilesystemKeys(checker);
+    assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
+    assertThat(diff.changedKeysWithNewValues()).isEmpty();
+  }
+
+  public void checkDirtyActions(BatchStat batchStatter, boolean forceDigests) throws Exception {
+    Artifact out1 = createDerivedArtifact("fiz");
+    Artifact out2 = createDerivedArtifact("pop");
+
+    FileSystemUtils.writeContentAsLatin1(out1.getPath(), "hello");
+    FileSystemUtils.writeContentAsLatin1(out2.getPath(), "fizzlepop");
+
+    Action action1 =
+        new TestAction(
+            Runnables.doNothing(), ImmutableSet.<Artifact>of(), ImmutableSet.<Artifact>of(out1));
+    Action action2 =
+        new TestAction(
+            Runnables.doNothing(), ImmutableSet.<Artifact>of(), ImmutableSet.<Artifact>of(out2));
+    differencer.inject(
+        ImmutableMap.<SkyKey, SkyValue>of(
+            ActionExecutionValue.key(action1), actionValue(action1, forceDigests),
+            ActionExecutionValue.key(action2), actionValue(action2, forceDigests)));
+    assertFalse(
+        driver
+            .evaluate(ImmutableList.<SkyKey>of(), false, 1, NullEventHandler.INSTANCE)
+            .hasError());
+    assertThat(new FilesystemValueChecker(evaluator, tsgm, null).getDirtyActionValues(batchStatter))
+        .isEmpty();
+
+    FileSystemUtils.writeContentAsLatin1(out1.getPath(), "goodbye");
+    assertEquals(
+        ActionExecutionValue.key(action1),
+        Iterables.getOnlyElement(
+            new FilesystemValueChecker(evaluator, tsgm, null).getDirtyActionValues(batchStatter)));
+  }
+
+  private Artifact createDerivedArtifact(String relPath) throws IOException {
+    Path outputPath = fs.getPath("/bin");
+    outputPath.createDirectory();
+    return new Artifact(
+        outputPath.getRelative(relPath), Root.asDerivedRoot(fs.getPath("/"), outputPath));
+  }
+
+  public void testDirtyActions() throws Exception {
+    checkDirtyActions(null, false);
+  }
+
+  public void testDirtyActionsBatchStat() throws Exception {
+    checkDirtyActions(
+        new BatchStat() {
+          @Override
+          public List<FileStatusWithDigest> batchStat(
+              boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
+              throws IOException {
+            List<FileStatusWithDigest> stats = new ArrayList<>();
+            for (PathFragment pathFrag : paths) {
+              stats.add(
+                  FileStatusWithDigestAdapter.adapt(
+                      fs.getRootDirectory().getRelative(pathFrag).statIfFound(Symlinks.NOFOLLOW)));
+            }
+            return stats;
+          }
+        },
+        false);
+  }
+
+  public void testDirtyActionsBatchStatWithDigest() throws Exception {
+    checkDirtyActions(
+        new BatchStat() {
+          @Override
+          public List<FileStatusWithDigest> batchStat(
+              boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
+              throws IOException {
+            List<FileStatusWithDigest> stats = new ArrayList<>();
+            for (PathFragment pathFrag : paths) {
+              final Path path = fs.getRootDirectory().getRelative(pathFrag);
+              stats.add(statWithDigest(path, path.statIfFound(Symlinks.NOFOLLOW)));
+            }
+            return stats;
+          }
+        },
+        true);
+  }
+
+  public void testDirtyActionsBatchStatFallback() throws Exception {
+    checkDirtyActions(
+        new BatchStat() {
+          @Override
+          public List<FileStatusWithDigest> batchStat(
+              boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
+              throws IOException {
+            throw new IOException("try again");
+          }
+        },
+        false);
+  }
+
+  private ActionExecutionValue actionValue(Action action, boolean forceDigest) {
+    Map<Artifact, FileValue> artifactData = new HashMap<>();
+    for (Artifact output : action.getOutputs()) {
+      try {
+        Path path = output.getPath();
+        FileStatusWithDigest stat =
+            forceDigest ? statWithDigest(path, path.statIfFound(Symlinks.NOFOLLOW)) : null;
+        artifactData.put(output, ActionMetadataHandler.fileValueFromArtifact(output, stat, tsgm));
+      } catch (IOException e) {
+        throw new IllegalStateException(e);
+      }
+    }
+    return new ActionExecutionValue(artifactData, ImmutableMap.<Artifact, FileArtifactValue>of());
+  }
+
+  public void testPropagatesRuntimeExceptions() throws Exception {
+    Collection<SkyKey> values =
+        ImmutableList.of(FileValue.key(RootedPath.toRootedPath(pkgRoot, new PathFragment("foo"))));
+    driver.evaluate(
+        values, false, SkyframeExecutor.DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
+    FilesystemValueChecker checker = new FilesystemValueChecker(evaluator, tsgm, null);
+
+    assertEmptyDiff(getDirtyFilesystemKeys(checker));
+
+    fs.statThrowsRuntimeException = true;
+    try {
+      getDirtyFilesystemKeys(checker);
+      fail();
+    } catch (RuntimeException e) {
+      assertThat(e).hasMessage("bork");
+    }
+  }
+
+  private static void assertEmptyDiff(Diff diff) {
+    assertDiffWithNewValues(diff);
+  }
+
+  private static void assertDiffWithNewValues(Diff diff, SkyKey... keysWithNewValues) {
+    assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
+    assertThat(diff.changedKeysWithNewValues().keySet())
+        .containsExactlyElementsIn(Arrays.asList(keysWithNewValues));
+  }
+
+  private class MockFileSystem extends InMemoryFileSystem {
+
+    boolean statThrowsRuntimeException;
+    boolean readlinkThrowsIoException;
+
+    MockFileSystem() {
+      super();
+    }
+
+    @Override
+    public FileStatus stat(Path path, boolean followSymlinks) throws IOException {
+      if (statThrowsRuntimeException) {
+        throw new RuntimeException("bork");
+      }
+      return super.stat(path, followSymlinks);
+    }
+
+    @Override
+    protected PathFragment readSymbolicLink(Path path) throws IOException {
+      if (readlinkThrowsIoException) {
+        throw new IOException("readlink failed");
+      }
+      return super.readSymbolicLink(path);
+    }
+  }
+
+  private static FileStatusWithDigest statWithDigest(final Path path, final FileStatus stat) {
+    return new FileStatusWithDigest() {
+      @Nullable
+      @Override
+      public byte[] getDigest() throws IOException {
+        return path.getMD5Digest();
+      }
+
+      @Override
+      public boolean isFile() {
+        return stat.isFile();
+      }
+
+      @Override
+      public boolean isSpecialFile() {
+        return stat.isSpecialFile();
+      }
+
+      @Override
+      public boolean isDirectory() {
+        return stat.isDirectory();
+      }
+
+      @Override
+      public boolean isSymbolicLink() {
+        return stat.isSymbolicLink();
+      }
+
+      @Override
+      public long getSize() throws IOException {
+        return stat.getSize();
+      }
+
+      @Override
+      public long getLastModifiedTime() throws IOException {
+        return stat.getLastModifiedTime();
+      }
+
+      @Override
+      public long getLastChangeTime() throws IOException {
+        return stat.getLastChangeTime();
+      }
+
+      @Override
+      public long getNodeId() throws IOException {
+        return stat.getNodeId();
+      }
+    };
+  }
+
+  private static Diff getDirtyFilesystemKeys(FilesystemValueChecker checker)
+      throws InterruptedException {
+    return checker.getDirtyKeys(new BasicFilesystemDirtinessChecker());
+  }
+}