Open source IncrementalLoadingTest.
--
MOS_MIGRATED_REVID=109795225
diff --git a/src/test/java/com/google/devtools/build/lib/pkgcache/IncrementalLoadingTest.java b/src/test/java/com/google/devtools/build/lib/pkgcache/IncrementalLoadingTest.java
new file mode 100644
index 0000000..935d042
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/pkgcache/IncrementalLoadingTest.java
@@ -0,0 +1,514 @@
+// Copyright 2015 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.pkgcache;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.ConstantRuleVisibility;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.NoSuchTargetException;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.skyframe.DiffAwareness;
+import com.google.devtools.build.lib.skyframe.PrecomputedValue;
+import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutor;
+import com.google.devtools.build.lib.skyframe.SkyValueDirtinessChecker;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.syntax.GlobList;
+import com.google.devtools.build.lib.testutil.ManualClock;
+import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.Dirent;
+import com.google.devtools.build.lib.vfs.FileStatus;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.ModifiedFileSet;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+
+import javax.annotation.Nullable;
+
+/**
+ * Tests for incremental loading; these cover both normal operation and diff awareness, for which a
+ * list of modified / added / removed files is available.
+ */
+@RunWith(JUnit4.class)
+public class IncrementalLoadingTest {
+ protected PackageCacheTester tester;
+
+ private Path throwOnReaddir = null;
+ private Path throwOnStat = null;
+
+ @Before
+ public final void createTester() throws Exception {
+ ManualClock clock = new ManualClock();
+ FileSystem fs =
+ new InMemoryFileSystem(clock) {
+ @Override
+ public Collection<Dirent> readdir(Path path, boolean followSymlinks) throws IOException {
+ if (path.equals(throwOnReaddir)) {
+ throw new FileNotFoundException(path.getPathString());
+ }
+ return super.readdir(path, followSymlinks);
+ }
+
+ @Nullable
+ @Override
+ public FileStatus stat(Path path, boolean followSymlinks) throws IOException {
+ if (path.equals(throwOnStat)) {
+ throw new IOException("bork " + path.getPathString());
+ }
+ return super.stat(path, followSymlinks);
+ }
+ };
+ tester = createTester(fs, clock);
+ }
+
+ protected PackageCacheTester createTester(FileSystem fs, ManualClock clock) throws Exception {
+ return new PackageCacheTester(fs, clock, Preprocessor.Factory.Supplier.NullSupplier.INSTANCE);
+ }
+
+ @Test
+ public void testNoChange() throws Exception {
+ tester.addFile("base/BUILD",
+ "filegroup(name = 'hello', srcs = ['foo.txt'])");
+ tester.sync();
+ Target oldTarget = tester.getTarget("//base:hello");
+ assertNotNull(oldTarget);
+
+ tester.sync();
+ Target newTarget = tester.getTarget("//base:hello");
+ assertSame(oldTarget, newTarget);
+ }
+
+ @Test
+ public void testModifyBuildFile() throws Exception {
+ tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
+ tester.sync();
+ Target oldTarget = tester.getTarget("//base:hello");
+
+ tester.modifyFile("base/BUILD", "filegroup(name = 'hello', srcs = ['bar.txt'])");
+ tester.sync();
+ Target newTarget = tester.getTarget("//base:hello");
+ assertNotSame(oldTarget, newTarget);
+ }
+
+ @Test
+ public void testModifyNonBuildFile() throws Exception {
+ tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
+ tester.addFile("base/foo.txt", "nothing");
+ tester.sync();
+ Target oldTarget = tester.getTarget("//base:hello");
+
+ tester.modifyFile("base/foo.txt", "other");
+ tester.sync();
+ Target newTarget = tester.getTarget("//base:hello");
+ assertSame(oldTarget, newTarget);
+ }
+
+ @Test
+ public void testRemoveNonBuildFile() throws Exception {
+ tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
+ tester.addFile("base/foo.txt", "nothing");
+ tester.sync();
+ Target oldTarget = tester.getTarget("//base:hello");
+
+ tester.removeFile("base/foo.txt");
+ tester.sync();
+ Target newTarget = tester.getTarget("//base:hello");
+ assertSame(oldTarget, newTarget);
+ }
+
+ @Test
+ public void testModifySymlinkedFileSamePackage() throws Exception {
+ tester.addSymlink("base/BUILD", "mybuild");
+ tester.addFile("base/mybuild", "filegroup(name = 'hello', srcs = ['foo.txt'])");
+ tester.sync();
+ Target oldTarget = tester.getTarget("//base:hello");
+ tester.modifyFile("base/mybuild", "filegroup(name = 'hello', srcs = ['bar.txt'])");
+ tester.sync();
+ Target newTarget = tester.getTarget("//base:hello");
+ assertNotSame(oldTarget, newTarget);
+ }
+
+ @Test
+ public void testModifySymlinkedFileDifferentPackage() throws Exception {
+ tester.addSymlink("base/BUILD", "../other/BUILD");
+ tester.addFile("other/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
+ tester.sync();
+ Target oldTarget = tester.getTarget("//base:hello");
+
+ tester.modifyFile("other/BUILD", "filegroup(name = 'hello', srcs = ['bar.txt'])");
+ tester.sync();
+ Target newTarget = tester.getTarget("//base:hello");
+ assertNotSame(oldTarget, newTarget);
+ }
+
+ @Test
+ public void testBUILDSymlinkModifiedThenChanges() throws Exception {
+ // We need to ensure that the timestamps of "one" and "two" are different, because Blaze
+ // currently does not recognize changes to symlinks if the timestamps of the old and the new
+ // file pointed to by the symlink are the same.
+ tester.addFile("one", "filegroup(name='a', srcs=['1'])");
+ tester.sync();
+
+ tester.addFile("two", "filegroup(name='a', srcs=['2'])");
+ tester.addSymlink("oldlink", "one");
+ tester.addSymlink("newlink", "one");
+ tester.addSymlink("a/BUILD", "../oldlink");
+ tester.sync();
+ Target a1 = tester.getTarget("//a:a");
+
+ tester.modifySymlink("a/BUILD", "../newlink");
+ tester.sync();
+
+ tester.getTarget("//a:a");
+
+ tester.modifySymlink("newlink", "two");
+ tester.sync();
+
+ Target a3 = tester.getTarget("//a:a");
+ assertNotSame(a1, a3);
+ }
+
+ @Test
+ public void testBUILDFileIsExternalSymlinkAndChanges() throws Exception {
+ tester.addFile("/nonroot/file", "filegroup(name='a', srcs=['file'])");
+ tester.addSymlink("a/BUILD", "/nonroot/file");
+ tester.sync();
+
+ Target a1 = tester.getTarget("//a:a");
+ tester.modifyFile("/nonroot/file", "filegroup(name='a', srcs=['file2'])");
+ tester.sync();
+
+ Target a2 = tester.getTarget("//a:a");
+ tester.sync();
+
+ assertNotSame(a1, a2);
+ }
+
+ @Test
+ public void testLabelWithTwoSegmentsAndTotalInvalidation() throws Exception {
+ tester.addFile("a/BUILD", "filegroup(name='fg', srcs=['b/c'])");
+ tester.addFile("a/b/BUILD");
+ tester.sync();
+
+ Target fg1 = tester.getTarget("//a:fg");
+ tester.everythingModified();
+ tester.sync();
+
+ Target fg2 = tester.getTarget("//a:fg");
+ assertSame(fg1, fg2);
+ }
+
+ @Test
+ public void testAddGlobFile() throws Exception {
+ tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = glob(['*.txt']))");
+ tester.addFile("base/foo.txt", "nothing");
+ tester.sync();
+ Target oldTarget = tester.getTarget("//base:hello");
+
+ tester.addFile("base/bar.txt", "also nothing");
+ tester.sync();
+ Target newTarget = tester.getTarget("//base:hello");
+ assertNotSame(oldTarget, newTarget);
+ }
+
+ @Test
+ public void testRemoveGlobFile() throws Exception {
+ tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = glob(['*.txt']))");
+ tester.addFile("base/foo.txt", "nothing");
+ tester.addFile("base/bar.txt", "also nothing");
+ tester.sync();
+ Target oldTarget = tester.getTarget("//base:hello");
+
+ tester.removeFile("base/bar.txt");
+ tester.sync();
+ Target newTarget = tester.getTarget("//base:hello");
+ assertNotSame(oldTarget, newTarget);
+ }
+
+ @Test
+ public void testPackageNotInLastBuildReplaced() throws Exception {
+ tester.addFile("a/BUILD", "filegroup(name='a', srcs=['bad.sh'])");
+ tester.sync();
+ Target a1 = tester.getTarget("//a:a");
+
+ tester.addFile("b/BUILD", "filegroup(name='b', srcs=['b.sh'])");
+ tester.modifyFile("a/BUILD", "filegroup(name='a', srcs=['good.sh'])");
+ tester.sync();
+ tester.getTarget("//b:b");
+
+ tester.sync();
+ Target a2 = tester.getTarget("//a:a");
+ assertNotSame(a1, a2);
+ }
+
+ @Test
+ public void testBrokenSymlinkAddedThenFixed() throws Exception {
+ tester.addFile("a/BUILD", "filegroup(name='a', srcs=glob(['**']))");
+ tester.sync();
+ Target a1 = tester.getTarget("//a:a");
+
+ tester.addSymlink("a/b", "../c");
+ tester.sync();
+ tester.getTarget("//a:a");
+
+ tester.addFile("c");
+ tester.sync();
+ Target a3 = tester.getTarget("//a:a");
+ assertNotSame(a1, a3);
+ }
+
+ @Test
+ public void testBuildFileWithSyntaxError() throws Exception {
+ tester.addFile("a/BUILD", "sh_library(xyz='a')");
+ tester.sync();
+ try {
+ tester.getTarget("//a:a");
+ fail();
+ } catch (NoSuchThingException e) {
+ // Expected
+ }
+
+ tester.modifyFile("a/BUILD", "sh_library(name='a')");
+ tester.sync();
+ tester.getTarget("//a:a");
+ }
+
+ @Test
+ public void testSymlinkedBuildFileWithSyntaxError() throws Exception {
+ tester.addFile("a/BUILD.real", "sh_library(xyz='a')");
+ tester.addSymlink("a/BUILD", "BUILD.real");
+ tester.sync();
+ try {
+ tester.getTarget("//a:a");
+ fail();
+ } catch (NoSuchThingException e) {
+ // Expected
+ }
+ tester.modifyFile("a/BUILD.real", "sh_library(name='a')");
+ tester.sync();
+ tester.getTarget("//a:a");
+ }
+
+ @Test
+ public void testTransientErrorsInGlobbing() throws Exception {
+ Path buildFile = tester.addFile("e/BUILD", "sh_library(name = 'e', data = glob(['*.txt']))");
+ Path parentDir = buildFile.getParentDirectory();
+ tester.addFile("e/data.txt");
+ throwOnReaddir = parentDir;
+ tester.sync();
+ Target target = tester.getTarget("//e:e");
+ assertThat(((Rule) target).containsErrors()).isTrue();
+ GlobList<?> globList = (GlobList<?>) ((Rule) target).getAttributeContainer().getAttr("data");
+ assertThat(globList).isEmpty();
+ throwOnReaddir = null;
+ tester.sync();
+ target = tester.getTarget("//e:e");
+ assertThat(((Rule) target).containsErrors()).isFalse();
+ globList = (GlobList<?>) ((Rule) target).getAttributeContainer().getAttr("data");
+ assertThat(globList).containsExactly(Label.parseAbsolute("//e:data.txt"));
+ }
+
+ @Test
+ public void testIrrelevantFileInSubdirDoesntReloadPackage() throws Exception {
+ tester.addFile("pkg/BUILD", "sh_library(name = 'pkg', srcs = glob(['**/*.sh']))");
+ tester.addFile("pkg/pkg.sh", "#!/bin/bash");
+ tester.addFile("pkg/bar/bar.sh", "#!/bin/bash");
+ Package pkg = tester.getTarget("//pkg:pkg").getPackage();
+
+ // Write file in directory to force reload of top-level glob.
+ tester.addFile("pkg/irrelevant_file");
+ tester.addFile("pkg/bar/irrelevant_file"); // Subglob is also reloaded.
+ assertSame(pkg, tester.getTarget("//pkg:pkg").getPackage());
+ }
+
+ @Test
+ public void testMissingPackages() throws Exception {
+ tester.sync();
+
+ try {
+ tester.getTarget("//a:a");
+ fail();
+ } catch (NoSuchThingException e) {
+ // expected
+ }
+
+ tester.addFile("a/BUILD", "sh_library(name='a')");
+ tester.sync();
+ tester.getTarget("//a:a");
+ }
+
+ static class PackageCacheTester {
+ private final ManualClock clock;
+ private final Path workspace;
+ private final Path outputBase;
+ private final Reporter reporter = new Reporter();
+ private final SkyframeExecutor skyframeExecutor;
+ private final List<Path> changes = new ArrayList<>();
+ private boolean everythingModified = false;
+
+ public PackageCacheTester(
+ FileSystem fs, ManualClock clock, Preprocessor.Factory.Supplier supplier)
+ throws IOException {
+ this.clock = clock;
+ workspace = fs.getPath("/workspace");
+ workspace.createDirectory();
+ outputBase = fs.getPath("/output_base");
+ outputBase.createDirectory();
+ addFile("WORKSPACE");
+
+ skyframeExecutor =
+ SequencedSkyframeExecutor.create(
+ new PackageFactory(TestRuleClassProvider.getRuleClassProvider()),
+ new TimestampGranularityMonitor(BlazeClock.instance()),
+ new BlazeDirectories(fs.getPath("/install"), fs.getPath("/output"), workspace),
+ null, /* BinTools */
+ null, /* workspaceStatusActionFactory */
+ TestRuleClassProvider.getRuleClassProvider().getBuildInfoFactories(),
+ ImmutableList.<DiffAwareness.Factory>of(),
+ Predicates.<PathFragment>alwaysFalse(),
+ supplier,
+ ImmutableMap.<SkyFunctionName, SkyFunction>of(),
+ ImmutableList.<PrecomputedValue.Injected>of(),
+ ImmutableList.<SkyValueDirtinessChecker>of());
+ skyframeExecutor.preparePackageLoading(
+ new PathPackageLocator(outputBase, ImmutableList.of(workspace)),
+ ConstantRuleVisibility.PUBLIC, true, 7, "",
+ UUID.randomUUID());
+ }
+
+ Path addFile(String fileName, String... content) throws IOException {
+ Path buildFile = workspace.getRelative(fileName);
+ Preconditions.checkState(!buildFile.exists());
+ Path currentPath = buildFile;
+
+ // Add the new file and all the directories that will be created by
+ // createDirectoryAndParents()
+ while (!currentPath.exists()) {
+ changes.add(currentPath);
+ currentPath = currentPath.getParentDirectory();
+ }
+
+ FileSystemUtils.createDirectoryAndParents(buildFile.getParentDirectory());
+ FileSystemUtils.writeContentAsLatin1(buildFile, Joiner.on('\n').join(content));
+ return buildFile;
+ }
+
+ void addSymlink(String fileName, String target) throws IOException {
+ Path path = workspace.getRelative(fileName);
+ Preconditions.checkState(!path.exists());
+ FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
+ path.createSymbolicLink(new PathFragment(target));
+ changes.add(path);
+ }
+
+ void removeFile(String fileName) throws IOException {
+ Path path = workspace.getRelative(fileName);
+ Preconditions.checkState(path.delete());
+ changes.add(path);
+ }
+
+ void modifyFile(String fileName, String... content) throws IOException {
+ Path path = workspace.getRelative(fileName);
+ Preconditions.checkState(path.exists());
+ Preconditions.checkState(path.delete());
+ FileSystemUtils.writeContentAsLatin1(path, Joiner.on('\n').join(content));
+ changes.add(path);
+ }
+
+ void modifySymlink(String fileName, String newTarget) throws IOException {
+ Path symlink = workspace.getRelative(fileName);
+ Preconditions.checkState(symlink.exists());
+ symlink.delete();
+ symlink.createSymbolicLink(new PathFragment(newTarget));
+ changes.add(symlink);
+ }
+
+ void everythingModified() {
+ everythingModified = true;
+ }
+
+ private ModifiedFileSet getModifiedFileSet() {
+ if (everythingModified) {
+ everythingModified = false;
+ return ModifiedFileSet.EVERYTHING_MODIFIED;
+ }
+
+ ModifiedFileSet.Builder builder = ModifiedFileSet.builder();
+ for (Path path : changes) {
+ if (!path.startsWith(workspace)) {
+ continue;
+ }
+
+ PathFragment workspacePath = path.relativeTo(workspace);
+ builder.modify(workspacePath);
+ }
+ return builder.build();
+ }
+
+ void sync() throws InterruptedException {
+ clock.advanceMillis(1);
+
+ skyframeExecutor.preparePackageLoading(
+ new PathPackageLocator(outputBase, ImmutableList.of(workspace)),
+ ConstantRuleVisibility.PUBLIC, true, 7, "",
+ UUID.randomUUID());
+ skyframeExecutor.invalidateFilesUnderPathForTesting(
+ new Reporter(), getModifiedFileSet(), workspace);
+ ((SequencedSkyframeExecutor) skyframeExecutor).handleDiffs(new Reporter());
+
+ changes.clear();
+ }
+
+ Target getTarget(String targetName)
+ throws NoSuchPackageException, NoSuchTargetException, InterruptedException {
+ Label label = Label.parseAbsoluteUnchecked(targetName);
+ return skyframeExecutor.getPackageManager().getTarget(reporter, label);
+ }
+ }
+}
