src/test/java/com/google/devtools/build/lib/skyframe/MacOSXFsEventsDiffAwarenessTest.java - bazel - Git at Google

 // Copyright 2016 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package com.google.devtools.build.lib.skyframe;

 import static org.junit.Assume.assumeFalse;

 import com.google.common.base.Charsets;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Multimap;
 import com.google.common.flogger.GoogleLogger;
 import com.google.devtools.build.lib.skyframe.DiffAwareness.View;
 import com.google.devtools.build.lib.skyframe.LocalDiffAwareness.Options;
 import com.google.devtools.build.lib.vfs.ModifiedFileSet;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.common.options.OptionsBase;
 import com.google.devtools.common.options.OptionsProvider;
 import java.io.IOException;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicReference;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;

 /** Tests for {@link MacOSXFsEventsDiffAwareness} */
 @RunWith(JUnit4.class)
 public class MacOSXFsEventsDiffAwarenessTest {
   private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();

   private static void rmdirs(Path directory) throws IOException {
     Files.walkFileTree(
         directory,
         new SimpleFileVisitor<Path>() {
           @Override
           public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
               throws IOException {
             Files.delete(file);
             return FileVisitResult.CONTINUE;
           }

           @Override
           public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
             Files.delete(dir);
             return FileVisitResult.CONTINUE;
           }
         });
   }

   private MacOSXFsEventsDiffAwareness underTest;
   private Path watchedPath;
   private OptionsProvider watchFsEnabledProvider;

   @Before
   public void setUp() throws Exception {
     watchedPath = com.google.common.io.Files.createTempDir().getCanonicalFile().toPath();
     underTest = new MacOSXFsEventsDiffAwareness(watchedPath.toString());
     LocalDiffAwareness.Options localDiffOptions = new LocalDiffAwareness.Options();
     localDiffOptions.watchFS = true;
     watchFsEnabledProvider = new LocalDiffAwarenessOptionsProvider(localDiffOptions);
   }

   @After
   public void tearDown() throws Exception {
     underTest.close();
     rmdirs(watchedPath);
   }

   private void scratchDir(String path) throws IOException {
     Path p = watchedPath.resolve(path);
     p.toFile().mkdirs();
   }

   private void scratchFile(String path, String contents) throws IOException {
     Path p = watchedPath.resolve(path);
     com.google.common.io.Files.write(contents.getBytes(Charsets.UTF_8), p.toFile());
   }

   private void scratchFile(String path) throws IOException {
     scratchFile(path, "");
   }

   /**
    * Checks that the union of the diffs between the current view and each member of some consecutive
    * sequence of views is the specific set of given files.
    *
    * @param view1 the view to compare to
    * @param rawPaths the files to expect in the view
    * @return the new view
    */
   private View assertDiff(View view1, Iterable<String> rawPaths)
       throws IncompatibleViewException, BrokenDiffAwarenessException, InterruptedException {
     Set<PathFragment> allPaths = new HashSet<>();
     for (String path : rawPaths) {
       allPaths.add(PathFragment.create(path));
     }
     Set<PathFragment> pathsYetToBeSeen = new HashSet<>(allPaths);

     // fsevents may be delayed (especially under machine load), which means that we may not notice
     // all file system changes in one go. Try enough times (multiple seconds) for the events to be
     // delivered. Given that each time we call getCurrentView we may get a subset of the total
     // events we expect, track the events we have already seen by subtracting them from the
     // pathsYetToBeSeen set.
     int attempts = 0;
     for (; ; ) {
       View view2 = underTest.getCurrentView(watchFsEnabledProvider);

       ModifiedFileSet diff = underTest.getDiff(view1, view2);
       // If fsevents lost events (e.g. because we weren't fast enough processing them or because
       // too many happened at the same time), there is nothing we can do. Yes, this means that if
       // our fsevents monitor always returns "everything modified", we aren't really testing
       // anything here... but let's assume we don't have such an obvious bug...
       assumeFalse("Lost events; diff unknown", diff.equals(ModifiedFileSet.EVERYTHING_MODIFIED));

       ImmutableSet<PathFragment> modifiedSourceFiles = diff.modifiedSourceFiles();
       allPaths.removeAll(modifiedSourceFiles);
       pathsYetToBeSeen.removeAll(modifiedSourceFiles);
       if (pathsYetToBeSeen.isEmpty()) {
         // Found all paths that we wanted to see as modified so now check that we didn't get any
         // extra paths we did not expect.
         if (!allPaths.isEmpty()) {
           throw new AssertionError("Paths " + allPaths + " unexpectedly reported as modified");
         }
         return view2;
       }

       if (attempts == 600) {
         throw new AssertionError("Paths " + pathsYetToBeSeen + " not found as modified");
       }
       logger.atInfo().log("Still have to see %d paths", pathsYetToBeSeen.size());
       Thread.sleep(100);
       attempts++;
       view1 = view2; // getDiff requires views to be sequential if we want to get meaningful data.
     }
   }

   @Test
   @Ignore("Test is flaky; see https://github.com/bazelbuild/bazel/issues/10776")
   public void testSimple() throws Exception {
     View view1 = underTest.getCurrentView(watchFsEnabledProvider);

     scratchDir("a/b");
     scratchFile("a/b/c");
     scratchDir("b/c");
     scratchFile("b/c/d");
     View view2 = assertDiff(view1, Arrays.asList("a", "a/b", "a/b/c", "b", "b/c", "b/c/d"));

     rmdirs(watchedPath.resolve("a"));
     rmdirs(watchedPath.resolve("b"));
     assertDiff(view2, Arrays.asList("a", "a/b", "a/b/c", "b", "b/c", "b/c/d"));
   }

   @Test
   @Ignore("Test is flaky; see https://github.com/bazelbuild/bazel/issues/10776")
   public void testRenameDirectory() throws Exception {
     scratchDir("dir1");
     scratchFile("dir1/file.c", "first");
     scratchDir("dir2");
     scratchFile("dir2/file.c", "second");
     View view1 = underTest.getCurrentView(watchFsEnabledProvider);

     Files.move(watchedPath.resolve("dir1"), watchedPath.resolve("dir3"));
     Files.move(watchedPath.resolve("dir2"), watchedPath.resolve("dir1"));
     assertDiff(
         view1, Arrays.asList("dir1", "dir1/file.c", "dir2", "dir2/file.c", "dir3", "dir3/file.c"));
   }

   @Test
   @Ignore("Test is flaky; see https://github.com/bazelbuild/bazel/issues/10776")
   public void testStress() throws Exception {
     View view1 = underTest.getCurrentView(watchFsEnabledProvider);

     // Attempt to cause fsevents to drop events by performing a lot of concurrent file accesses
     // which then may result in our own callback in fsevents.cc not being able to keep up.
     // There is no guarantee that we'll trigger this condition, but on 2020-02-28 on a Mac Pro
     // 2013, this happened pretty predictably with the settings below.
     logger.atInfo().log("Starting file creation under %s", watchedPath);
     ExecutorService executor = Executors.newCachedThreadPool();
     int nThreads = 100;
     int nFilesPerThread = 100;
     Multimap<String, String> dirToFilesToCreate = HashMultimap.create();
     for (int i = 0; i < nThreads; i++) {
       String dir = "" + i;
       for (int j = 0; j < nFilesPerThread; j++) {
         String file = dir + "/" + j;
         dirToFilesToCreate.put(dir, file);
       }
     }
     CountDownLatch latch = new CountDownLatch(nThreads);
     AtomicReference<IOException> firstError = new AtomicReference<>(null);
     dirToFilesToCreate
         .asMap()
         .forEach(
             (dir, files) ->
                 executor.submit(
                     () -> {
                       try {
                         scratchDir(dir);
                         for (String file : files) {
                           scratchFile(file);
                         }
                       } catch (IOException e) {
                         firstError.compareAndSet(null, e);
                       }
                       latch.countDown();
                     }));
     latch.await();
     executor.shutdown();
     IOException e = firstError.get();
     if (e != null) {
       throw e;
     }

     assertDiff(view1, Iterables.concat(dirToFilesToCreate.keySet(), dirToFilesToCreate.values()));
   }

   /**
    * Only returns a fixed options class for {@link LocalDiffAwareness.Options}.
    */
   private static final class LocalDiffAwarenessOptionsProvider implements OptionsProvider {
     private final Options localDiffOptions;

     private LocalDiffAwarenessOptionsProvider(Options localDiffOptions) {
       this.localDiffOptions = localDiffOptions;
     }

     @Override
     public <O extends OptionsBase> O getOptions(Class<O> optionsClass) {
       if (optionsClass.equals(LocalDiffAwareness.Options.class)) {
         return optionsClass.cast(localDiffOptions);
       }
       return null;
     }

     @Override
     public Map<String, Object> getStarlarkOptions() {
       return ImmutableMap.of();
     }
   }
 }
	// Copyright 2016 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package com.google.devtools.build.lib.skyframe;

	import static org.junit.Assume.assumeFalse;

	import com.google.common.base.Charsets;
	import com.google.common.collect.HashMultimap;
	import com.google.common.collect.ImmutableMap;
	import com.google.common.collect.ImmutableSet;
	import com.google.common.collect.Iterables;
	import com.google.common.collect.Multimap;
	import com.google.common.flogger.GoogleLogger;
	import com.google.devtools.build.lib.skyframe.DiffAwareness.View;
	import com.google.devtools.build.lib.skyframe.LocalDiffAwareness.Options;
	import com.google.devtools.build.lib.vfs.ModifiedFileSet;
	import com.google.devtools.build.lib.vfs.PathFragment;
	import com.google.devtools.common.options.OptionsBase;
	import com.google.devtools.common.options.OptionsProvider;
	import java.io.IOException;
	import java.nio.file.FileVisitResult;
	import java.nio.file.Files;
	import java.nio.file.Path;
	import java.nio.file.SimpleFileVisitor;
	import java.nio.file.attribute.BasicFileAttributes;
	import java.util.Arrays;
	import java.util.HashSet;
	import java.util.Map;
	import java.util.Set;
	import java.util.concurrent.CountDownLatch;
	import java.util.concurrent.ExecutorService;
	import java.util.concurrent.Executors;
	import java.util.concurrent.atomic.AtomicReference;
	import org.junit.After;
	import org.junit.Before;
	import org.junit.Ignore;
	import org.junit.Test;
	import org.junit.runner.RunWith;
	import org.junit.runners.JUnit4;

	/** Tests for {@link MacOSXFsEventsDiffAwareness} */
	@RunWith(JUnit4.class)
	public class MacOSXFsEventsDiffAwarenessTest {
	private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();

	private static void rmdirs(Path directory) throws IOException {
	Files.walkFileTree(
	directory,
	new SimpleFileVisitor<Path>() {
	@Override
	public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
	throws IOException {
	Files.delete(file);
	return FileVisitResult.CONTINUE;
	}

	@Override
	public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
	Files.delete(dir);
	return FileVisitResult.CONTINUE;
	}
	});
	}

	private MacOSXFsEventsDiffAwareness underTest;
	private Path watchedPath;
	private OptionsProvider watchFsEnabledProvider;

	@Before
	public void setUp() throws Exception {
	watchedPath = com.google.common.io.Files.createTempDir().getCanonicalFile().toPath();
	underTest = new MacOSXFsEventsDiffAwareness(watchedPath.toString());
	LocalDiffAwareness.Options localDiffOptions = new LocalDiffAwareness.Options();
	localDiffOptions.watchFS = true;
	watchFsEnabledProvider = new LocalDiffAwarenessOptionsProvider(localDiffOptions);
	}

	@After
	public void tearDown() throws Exception {
	underTest.close();
	rmdirs(watchedPath);
	}

	private void scratchDir(String path) throws IOException {
	Path p = watchedPath.resolve(path);
	p.toFile().mkdirs();
	}

	private void scratchFile(String path, String contents) throws IOException {
	Path p = watchedPath.resolve(path);
	com.google.common.io.Files.write(contents.getBytes(Charsets.UTF_8), p.toFile());
	}

	private void scratchFile(String path) throws IOException {
	scratchFile(path, "");
	}

	/**
	* Checks that the union of the diffs between the current view and each member of some consecutive
	* sequence of views is the specific set of given files.
	*
	* @param view1 the view to compare to
	* @param rawPaths the files to expect in the view
	* @return the new view
	*/
	private View assertDiff(View view1, Iterable<String> rawPaths)
	throws IncompatibleViewException, BrokenDiffAwarenessException, InterruptedException {
	Set<PathFragment> allPaths = new HashSet<>();
	for (String path : rawPaths) {
	allPaths.add(PathFragment.create(path));
	}
	Set<PathFragment> pathsYetToBeSeen = new HashSet<>(allPaths);

	// fsevents may be delayed (especially under machine load), which means that we may not notice
	// all file system changes in one go. Try enough times (multiple seconds) for the events to be
	// delivered. Given that each time we call getCurrentView we may get a subset of the total
	// events we expect, track the events we have already seen by subtracting them from the
	// pathsYetToBeSeen set.
	int attempts = 0;
	for (; ; ) {
	View view2 = underTest.getCurrentView(watchFsEnabledProvider);

	ModifiedFileSet diff = underTest.getDiff(view1, view2);
	// If fsevents lost events (e.g. because we weren't fast enough processing them or because
	// too many happened at the same time), there is nothing we can do. Yes, this means that if
	// our fsevents monitor always returns "everything modified", we aren't really testing
	// anything here... but let's assume we don't have such an obvious bug...
	assumeFalse("Lost events; diff unknown", diff.equals(ModifiedFileSet.EVERYTHING_MODIFIED));

	ImmutableSet<PathFragment> modifiedSourceFiles = diff.modifiedSourceFiles();
	allPaths.removeAll(modifiedSourceFiles);
	pathsYetToBeSeen.removeAll(modifiedSourceFiles);
	if (pathsYetToBeSeen.isEmpty()) {
	// Found all paths that we wanted to see as modified so now check that we didn't get any
	// extra paths we did not expect.
	if (!allPaths.isEmpty()) {
	throw new AssertionError("Paths " + allPaths + " unexpectedly reported as modified");
	}
	return view2;
	}

	if (attempts == 600) {
	throw new AssertionError("Paths " + pathsYetToBeSeen + " not found as modified");
	}
	logger.atInfo().log("Still have to see %d paths", pathsYetToBeSeen.size());
	Thread.sleep(100);
	attempts++;
	view1 = view2; // getDiff requires views to be sequential if we want to get meaningful data.
	}
	}

	@Test
	@Ignore("Test is flaky; see https://github.com/bazelbuild/bazel/issues/10776")
	public void testSimple() throws Exception {
	View view1 = underTest.getCurrentView(watchFsEnabledProvider);

	scratchDir("a/b");
	scratchFile("a/b/c");
	scratchDir("b/c");
	scratchFile("b/c/d");
	View view2 = assertDiff(view1, Arrays.asList("a", "a/b", "a/b/c", "b", "b/c", "b/c/d"));

	rmdirs(watchedPath.resolve("a"));
	rmdirs(watchedPath.resolve("b"));
	assertDiff(view2, Arrays.asList("a", "a/b", "a/b/c", "b", "b/c", "b/c/d"));
	}

	@Test
	@Ignore("Test is flaky; see https://github.com/bazelbuild/bazel/issues/10776")
	public void testRenameDirectory() throws Exception {
	scratchDir("dir1");
	scratchFile("dir1/file.c", "first");
	scratchDir("dir2");
	scratchFile("dir2/file.c", "second");
	View view1 = underTest.getCurrentView(watchFsEnabledProvider);

	Files.move(watchedPath.resolve("dir1"), watchedPath.resolve("dir3"));
	Files.move(watchedPath.resolve("dir2"), watchedPath.resolve("dir1"));
	assertDiff(
	view1, Arrays.asList("dir1", "dir1/file.c", "dir2", "dir2/file.c", "dir3", "dir3/file.c"));
	}

	@Test
	@Ignore("Test is flaky; see https://github.com/bazelbuild/bazel/issues/10776")
	public void testStress() throws Exception {
	View view1 = underTest.getCurrentView(watchFsEnabledProvider);

	// Attempt to cause fsevents to drop events by performing a lot of concurrent file accesses
	// which then may result in our own callback in fsevents.cc not being able to keep up.
	// There is no guarantee that we'll trigger this condition, but on 2020-02-28 on a Mac Pro
	// 2013, this happened pretty predictably with the settings below.
	logger.atInfo().log("Starting file creation under %s", watchedPath);
	ExecutorService executor = Executors.newCachedThreadPool();
	int nThreads = 100;
	int nFilesPerThread = 100;
	Multimap<String, String> dirToFilesToCreate = HashMultimap.create();
	for (int i = 0; i < nThreads; i++) {
	String dir = "" + i;
	for (int j = 0; j < nFilesPerThread; j++) {
	String file = dir + "/" + j;
	dirToFilesToCreate.put(dir, file);
	}
	}
	CountDownLatch latch = new CountDownLatch(nThreads);
	AtomicReference<IOException> firstError = new AtomicReference<>(null);
	dirToFilesToCreate
	.asMap()
	.forEach(
	(dir, files) ->
	executor.submit(
	() -> {
	try {
	scratchDir(dir);
	for (String file : files) {
	scratchFile(file);
	}
	} catch (IOException e) {
	firstError.compareAndSet(null, e);
	}
	latch.countDown();
	}));
	latch.await();
	executor.shutdown();
	IOException e = firstError.get();
	if (e != null) {
	throw e;
	}

	assertDiff(view1, Iterables.concat(dirToFilesToCreate.keySet(), dirToFilesToCreate.values()));
	}

	/**
	* Only returns a fixed options class for {@link LocalDiffAwareness.Options}.
	*/
	private static final class LocalDiffAwarenessOptionsProvider implements OptionsProvider {
	private final Options localDiffOptions;

	private LocalDiffAwarenessOptionsProvider(Options localDiffOptions) {
	this.localDiffOptions = localDiffOptions;
	}

	@Override
	public <O extends OptionsBase> O getOptions(Class<O> optionsClass) {
	if (optionsClass.equals(LocalDiffAwareness.Options.class)) {
	return optionsClass.cast(localDiffOptions);
	}
	return null;
	}

	@Override
	public Map<String, Object> getStarlarkOptions() {
	return ImmutableMap.of();
	}
	}
	}