src/test/java/com/google/devtools/build/lib/skyframe/LocalDiffAwarenessTest.java

// Copyright 2022 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.cmdline.IgnoredSubdirectories;
import com.google.devtools.build.lib.skyframe.DiffAwareness.View;
import com.google.devtools.build.lib.skyframe.LocalDiffAwareness.SequentialView;
import com.google.devtools.build.lib.testing.common.FakeOptions;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.ModifiedFileSet;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.util.FileSystems;
import com.google.devtools.common.options.Options;
import com.google.devtools.common.options.OptionsProvider;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.Set;
import org.junit.After;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

/** Integration tests for LocalDiffAwareness. */
@RunWith(JUnit4.class)
public class LocalDiffAwarenessTest {

  /** Try this many times to pick up file changes. Inotify needs some nanoseconds of patience. */
  private static final int MAX_RETRY_COUNT = 20;

  private OptionsProvider watchFsEnabledProvider;
  private LocalDiffAwareness localDiff;
  private DiffAwareness.View oldView;
  private Path testCaseRoot;
  private Path testCaseIgnoredDir;

  @Rule public final TemporaryFolder tmp = new TemporaryFolder();

  @Before
  public final void initializeSettings() throws Exception {
    LocalDiffAwareness.Factory factory =
        new LocalDiffAwareness.Factory(
            ImmutableList.<String>of(), new FsEventsNativeDepsServiceImpl());
    FileSystem fileSystem = FileSystems.getNativeFileSystem(DigestHashFunction.SHA256);
    testCaseRoot = fileSystem.getPath(tmp.getRoot().getAbsolutePath());
    testCaseIgnoredDir = testCaseRoot.getChild("ignored-dir");
    testCaseIgnoredDir.createDirectoryAndParents();

    LocalDiffAwareness.Options localDiffOptions =
        Options.getDefaults(LocalDiffAwareness.Options.class);
    localDiffOptions.setWatchFS(true);
    watchFsEnabledProvider = FakeOptions.of(localDiffOptions);

    localDiff =
        (LocalDiffAwareness)
            factory.maybeCreate(
                Root.fromPath(testCaseRoot),
                IgnoredSubdirectories.of(
                    ImmutableSet.of(testCaseIgnoredDir.asFragment().toRelative())),
                watchFsEnabledProvider);

    // Ignore test failures when run on a Mac.
    //
    // On a Mac, LocalDiffAwareness.Factory#maybeCreate will produce a MacOSXFsEventsDiffAwareness.
    // There's a known issue with the underlying implementation
    // (https://github.com/bazelbuild/bazel/issues/10776); basically all the test cases in here
    // consistently fail, presumably due to the same underlying issue with FSEvents. Also,
    // MacOSXFsEventsDiffAwareness is already unit-tested separately anyway (although not very well
    // because of the bug).
    Assume.assumeFalse(OS.DARWIN.equals(OS.getCurrent()));
  }

  @After
  public final void closeLocalDiff() throws Exception {
    localDiff.close();
  }

  @After
  public final void deleteTestCaseRoot() throws Exception {
    testCaseRoot.deleteTree();
  }

  private void captureFirstView(OptionsProvider options) throws BrokenDiffAwarenessException {
    oldView = localDiff.getCurrentView(options);
    Preconditions.checkNotNull(localDiff);
  }

  @Test
  public void areInSequenceWithEverythingModifiedShouldAlwaysReturnFalse() throws Exception {
    captureFirstView(watchFsEnabledProvider);
    SequentialView old = (SequentialView) oldView;
    SequentialView everythingMod = (SequentialView) LocalDiffAwareness.EVERYTHING_MODIFIED;
    assertThat(LocalDiffAwareness.areInSequence(old, everythingMod)).isFalse();
    assertThat(LocalDiffAwareness.areInSequence(everythingMod, old)).isFalse();
    assertThat(LocalDiffAwareness.areInSequence(everythingMod, everythingMod)).isFalse();
  }

  @Test
  public void testFileAdded() throws Exception {
    captureFirstView(watchFsEnabledProvider);
    touch("foo.txt");
    new ModifiedFileSetChecker().modify("foo.txt").check();
  }

  @Test
  public void testSymlink() throws Exception {
    captureFirstView(watchFsEnabledProvider);
    touch("a");
    symlink("b", "a");
    new ModifiedFileSetChecker().modify("a").modify("b").check();
  }

  @Test
  public void testSymlinkBroken() throws Exception {
    captureFirstView(watchFsEnabledProvider);
    symlink("b", "a");
    new ModifiedFileSetChecker().modify("b").check();
  }

  @Test
  public void testFileModified() throws Exception {
    captureFirstView(watchFsEnabledProvider);

    touch("foo.txt");
    new ModifiedFileSetChecker().modify("foo.txt").check();

    new ModifiedFileSetChecker().check();

    touch("foo.txt");
    new ModifiedFileSetChecker().modify("foo.txt").check();
  }

  @Test
  public void testIgnoredFileModified() throws Exception {
    captureFirstView(watchFsEnabledProvider);

    touch(testCaseIgnoredDir.getRelative("foo").getPathString());
    new ModifiedFileSetChecker().check();
  }

  @Test
  public void testFileRemoved() throws Exception {
    captureFirstView(watchFsEnabledProvider);

    touch("foo.txt");
    new ModifiedFileSetChecker().modify("foo.txt").check();

    rm("foo.txt");
    new ModifiedFileSetChecker().modify("foo.txt").check();
  }

  @Test
  public void testFileAddAndRemove() throws Exception {
    captureFirstView(watchFsEnabledProvider);

    touch("foo.txt");
    touch("bar.txt");
    new ModifiedFileSetChecker().modify("foo.txt").modify("bar.txt").check();

    rm("foo.txt");
    touch("foo.txt");
    new ModifiedFileSetChecker().modify("foo.txt").check();
  }

  @Test
  public void testAddDirectory() throws Exception {
    captureFirstView(watchFsEnabledProvider);

    mkdir("equestria");
    touch("equestria/foo.txt");
    new ModifiedFileSetChecker().modify("equestria").modify("equestria/foo.txt").check();
  }

  @Test
  public void testRemoveDirectory() throws Exception {
    captureFirstView(watchFsEnabledProvider);

    mkdir("equestria");
    touch("equestria/foo.txt");
    new ModifiedFileSetChecker().modify("equestria").modify("equestria/foo.txt").check();

    rm("equestria");
    new ModifiedFileSetChecker().modify("equestria").modify("equestria/foo.txt").check();
  }

  @Test
  public void testMoveDirectory() throws Exception {
    captureFirstView(watchFsEnabledProvider);

    mkdir("equestria");
    touch("equestria/foo.txt");
    new ModifiedFileSetChecker().modify("equestria").modify("equestria/foo.txt").check();

    testCaseRoot.getRelative("equestria").renameTo(testCaseRoot.getRelative("equestria2"));
    // The contents of a moved directory are *not* reported as modified.
    new ModifiedFileSetChecker()
        .modify("equestria")
        .modify("equestria2")
        .modify("equestria2/foo.txt")
        .check();
  }

  // Regression test for https://github.com/bazelbuild/bazel/issues/22417.
  @Test
  public void testDirectoryWithSubdirectoryMovedOutAndRecreated() throws Exception {
    mkdir("dir/sub");
    captureFirstView(watchFsEnabledProvider);

    // Moving a directory out of the watched tree only produces a deletion event for "dir" itself,
    // so the watch key of "dir/sub" stays valid and keeps watching the moved-away inode.
    move("dir", "ignored-dir/moved");
    new ModifiedFileSetChecker().modify("dir").check();

    // Registering the new "dir/sub" yields a new watch key for a path that is still associated
    // with the old one. This used to crash with "IllegalArgumentException: value already present".
    mkdir("dir/sub");
    new ModifiedFileSetChecker().modify("dir").modify("dir/sub").check();

    // Only the new directory is watched under "dir/sub"; changes to the moved-away one are not
    // reported under its old path.
    touch("ignored-dir/moved/sub/stale.txt");
    touch("dir/sub/new.txt");
    new ModifiedFileSetChecker().modify("dir/sub/new.txt").check();
  }

  @Test
  public void testLotsOfChanges() throws Exception {
    captureFirstView(watchFsEnabledProvider);

    mkdir("pre");
    touch("pre/pre.txt");
    new ModifiedFileSetChecker().modify("pre").modify("pre/pre.txt").check();

    mkdir("a");
    touch("a/a1.txt");
    touch("a/a2.txt");
    rm("a");
    mkdir("a");
    touch("a/a1.txt");
    touch("a/a2.txt");
    mkdir("a/b");
    touch("a/b/b1.txt");
    new ModifiedFileSetChecker()
        .modify("a")
        .modify("a/a1.txt")
        .modify("a/a2.txt")
        .modify("a/b")
        .modify("a/b/b1.txt")
        .check();

    rm("a/b/b1.txt");
    touch("a/b/b2.txt");
    rm("a/b");
    mkdir("a/b");
    rm("a/b");
    mkdir("a/b");
    touch("a/b/b3.txt");
    new ModifiedFileSetChecker()
        .modify("a/b")
        .modify("a/b/b1.txt")
        .modify("a/b/b2.txt")
        .modify("a/b/b3.txt")
        .check();

    rm("a/b/b3.txt");
    new ModifiedFileSetChecker().modify("a/b/b3.txt").check();
  }

  private static OptionsProvider createWatchFsDisabledProvider() {
    final LocalDiffAwareness.Options localDiffOptions =
        Options.getDefaults(LocalDiffAwareness.Options.class);
    localDiffOptions.setWatchFS(false);
    return FakeOptions.of(localDiffOptions);
  }

  @Test
  public void testEnableWatchFs() throws Exception {
    OptionsProvider watchFsDisabledProvider = createWatchFsDisabledProvider();
    captureFirstView(watchFsDisabledProvider);

    new ModifiedFileSetChecker().checkEverythingModified(watchFsDisabledProvider);

    touch("a.txt");
    new ModifiedFileSetChecker().checkEverythingModified(watchFsDisabledProvider);

    touch("b.txt");
    new ModifiedFileSetChecker().checkEverythingModified(watchFsEnabledProvider);

    touch("c.txt");
    new ModifiedFileSetChecker().modify("c.txt").check();
  }

  @Test
  public void testDisableWatchFs() throws Exception {
    OptionsProvider watchFsDisabledProvider = createWatchFsDisabledProvider();
    captureFirstView(watchFsEnabledProvider);

    new ModifiedFileSetChecker().check();

    assertThrows(
        BrokenDiffAwarenessException.class,
        () -> localDiff.getCurrentView(watchFsDisabledProvider));
  }

  @Test
  public void testOverflow() throws Exception {
    captureFirstView(watchFsEnabledProvider);
    for (int i = 0; i < 600; i++) {
      touch("file" + i + ".txt");
    }
    // Give inotify and JVM poller a moment to queue events
    Thread.sleep(200);
    new ModifiedFileSetChecker().checkEverythingModified(watchFsEnabledProvider);

    // After recovering from overflow, incremental diffing works immediately on the next view.
    touch("single.txt");
    new ModifiedFileSetChecker().modify("single.txt").check();
  }

  @Test
  public void testDirectoryCreatedDuringOverflow() throws Exception {
    captureFirstView(watchFsEnabledProvider);
    mkdir("newdir");
    for (int i = 0; i < 600; i++) {
      touch("file" + i + ".txt");
    }
    Thread.sleep(200);
    new ModifiedFileSetChecker().checkEverythingModified(watchFsEnabledProvider);

    // Subdirectories created during overflow are discovered and registered,
    // so subsequent modifications inside them are tracked incrementally.
    touch("newdir/inner.txt");
    new ModifiedFileSetChecker().modify("newdir/inner.txt").check();
  }

  @Test
  public void testDirectoryDeletedDuringOverflow() throws Exception {
    mkdir("deldir");
    touch("deldir/f.txt");
    captureFirstView(watchFsEnabledProvider);

    rm("deldir");
    for (int i = 0; i < 600; i++) {
      touch("file" + i + ".txt");
    }
    Thread.sleep(200);
    new ModifiedFileSetChecker().checkEverythingModified(watchFsEnabledProvider);

    // Subsequent modifications to other files work incrementally without stale key issues.
    touch("foo.txt");
    new ModifiedFileSetChecker().modify("foo.txt").check();
  }

  @Test
  public void testDirectoryReplacedDuringOverflow() throws Exception {
    mkdir("dir");
    touch("dir/old.txt");
    captureFirstView(watchFsEnabledProvider);

    // Overflow the watch key of the root directory first, so that the move and the re-creation
    // below are among the events that are dropped.
    for (int i = 0; i < 600; i++) {
      touch("file" + i + ".txt");
    }
    // Move the watched directory out of sight and put a different directory in its place. The
    // watch key of the old directory stays valid, but it no longer watches what is at "dir" now.
    move("dir", "ignored-dir/moved");
    mkdir("dir");
    Thread.sleep(200);
    new ModifiedFileSetChecker().checkEverythingModified(watchFsEnabledProvider);

    // Overflow recovery must have registered the replacement directory.
    touch("dir/new.txt");
    new ModifiedFileSetChecker().modify("dir/new.txt").check();
  }

  @Test
  public void testDirectoryMovedAndReplacedDuringOverflow() throws Exception {
    mkdir("dir1");
    touch("dir1/old.txt");
    captureFirstView(watchFsEnabledProvider);

    // Overflow the watch key of the root directory first, so that the move and the re-creation
    // below are among the events that are dropped.
    for (int i = 0; i < 600; i++) {
      touch("file" + i + ".txt");
    }
    // Both the moved directory and its replacement remain under the watch root, so the watch key
    // of the moved directory must be rebound to its new path instead of being cancelled.
    move("dir1", "dir2");
    mkdir("dir1");
    Thread.sleep(200);
    new ModifiedFileSetChecker().checkEverythingModified(watchFsEnabledProvider);

    touch("dir1/new.txt");
    touch("dir2/moved.txt");
    new ModifiedFileSetChecker().modify("dir1/new.txt").modify("dir2/moved.txt").check();
  }

  @Test
  public void testUnreadableDirectory() throws Exception {
    mkdir("unreadable");
    Path unreadable = testCaseRoot.getRelative("unreadable");
    unreadable.chmod(0);
    try {
      boolean readable = true;
      try {
        var unused = unreadable.getDirectoryEntries();
      } catch (IOException e) {
        readable = false;
      }
      // Running as root bypasses directory permissions, in which case there is nothing to test.
      Assume.assumeFalse(readable);

      // A directory that cannot be watched must be reported instead of being silently skipped,
      // which would make all subsequent diffs miss changes under it.
      assertThrows(
          BrokenDiffAwarenessException.class,
          () -> localDiff.getCurrentView(watchFsEnabledProvider));
    } finally {
      unreadable.chmod(0755);
    }
  }

  @Test
  public void testUnreadableIgnoredDirectory() throws Exception {
    testCaseIgnoredDir.chmod(0);
    try {
      boolean readable = true;
      try {
        var unused = testCaseIgnoredDir.getDirectoryEntries();
      } catch (IOException e) {
        readable = false;
      }
      // Running as root bypasses directory permissions, in which case there is nothing to test.
      Assume.assumeFalse(readable);

      // Not being able to watch a directory that we were told to ignore is fine.
      captureFirstView(watchFsEnabledProvider);
      touch("foo.txt");
      new ModifiedFileSetChecker().modify("foo.txt").check();
    } finally {
      testCaseIgnoredDir.chmod(0755);
    }
  }

  @Test
  public void modifiedPathIsntUnderWatchRoot() {
    java.nio.file.Path otherRootDirectoryNioPath = Paths.get("/notundertestroot");
    assertThat(otherRootDirectoryNioPath.startsWith(Paths.get(testCaseRoot.getPathString())))
        .isFalse();

    View oldView =
        new LocalDiffAwareness.SequentialView(
            localDiff,
            /* position= */ 0,
            /* modifiedAbsolutePaths= */ ImmutableSet.of(),
            /* isOverflow= */ false);
    View newView =
        new LocalDiffAwareness.SequentialView(
            localDiff,
            /* position= */ 1,
            /* modifiedAbsolutePaths= */ ImmutableSet.of(
                otherRootDirectoryNioPath.resolve("foo.txt")),
            /* isOverflow= */ false);
    Throwable throwable =
        assertThrows(BrokenDiffAwarenessException.class, () -> localDiff.getDiff(oldView, newView));
    assertThat(throwable)
        .hasMessageThat()
        // Do a round-trip through PathFragment to deal with Windows path separators.
        .contains(
            PathFragment.create("/notundertestroot/foo.txt is not under ").getPathString()
                + PathFragment.create(testCaseRoot.getPathString()).getPathString());
  }

  private void touch(String pathString) throws IOException {
    Path path = testCaseRoot.getRelative(pathString);
    FileSystemUtils.createEmptyFile(path);
    FileSystemUtils.writeIsoLatin1(path, "Sunshine, sunshine, ladybugs awake!");
  }

  private void mkdir(String pathString) throws IOException {
    Path path = testCaseRoot.getRelative(pathString);
    path.createDirectoryAndParents();
  }

  private void rm(String pathString) throws IOException {
    Path path = testCaseRoot.getRelative(pathString);
    path.deleteTree();
  }

  private void move(String from, String to) throws IOException {
    testCaseRoot.getRelative(from).renameTo(testCaseRoot.getRelative(to));
  }

  private void symlink(String from, String to) throws IOException {
    Path fromPath = testCaseRoot.getRelative(from);
    Path toPath = testCaseRoot.getRelative(to);
    FileSystemUtils.ensureSymbolicLink(fromPath, toPath);
  }

  private class ModifiedFileSetChecker {
    private final Set<PathFragment> modified = new HashSet<>();

    void check() throws Exception {
      // Unfortunately, inotify needs a few milliseconds (more than a few in the worst case)
      // after a change to pick up a list of changed files. Trying a few times to make sure.
      for (int i = 0; i < MAX_RETRY_COUNT; i++) {
        Thread.sleep(150);
        DiffAwareness.View newView = localDiff.getCurrentView(watchFsEnabledProvider);
        ModifiedFileSet modifiedFileSet = localDiff.getDiff(oldView, newView);
        oldView = newView;
        assertThat(modifiedFileSet.treatEverythingAsModified()).isFalse();
        if (modifiedFileSet.modifiedSourceFiles().isEmpty()) {
          continue;
        }
        assertThat(modifiedFileSet.modifiedSourceFiles()).isEqualTo(modified);
        return;
      }
      // If we never received any changes, make sure this is what we actually expect.
      assertThat(modified).isEmpty();
    }

    void checkEverythingModified(OptionsProvider options) throws Exception {
      DiffAwareness.View newView = localDiff.getCurrentView(options);
      ModifiedFileSet modifiedFileSet = localDiff.getDiff(oldView, newView);
      oldView = newView;
      assertThat(modifiedFileSet.treatEverythingAsModified()).isTrue();
    }

    @CanIgnoreReturnValue
    ModifiedFileSetChecker modify(String filename) {
      modified.add(PathFragment.create(filename));
      return this;
    }
  }
}