blob: bf4ed0c4de0e40e13f1fcb37132552c23c122fcf [file] [log] [blame]
// Copyright 2015 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import com.google.common.base.Functions;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.testing.EqualsTester;
import com.google.devtools.build.lib.actions.FileStateValue;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.ServerDirectories;
import com.google.devtools.build.lib.analysis.util.AnalysisMock;
import com.google.devtools.build.lib.clock.BlazeClock;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.events.NullEventHandler;
import com.google.devtools.build.lib.io.FileSymlinkCycleUniquenessFunction;
import com.google.devtools.build.lib.io.InconsistentFilesystemException;
import com.google.devtools.build.lib.packages.Globber;
import com.google.devtools.build.lib.packages.Globber.Operation;
import com.google.devtools.build.lib.packages.RuleClassProvider;
import com.google.devtools.build.lib.packages.WorkspaceFileValue;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction;
import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction;
import com.google.devtools.build.lib.skyframe.GlobValue.InvalidGlobPatternException;
import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy;
import com.google.devtools.build.lib.testutil.ManualClock;
import com.google.devtools.build.lib.testutil.TestConstants;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.Dirent;
import com.google.devtools.build.lib.vfs.FileStateKey;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.lib.vfs.UnixGlob;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import com.google.devtools.build.skyframe.ErrorInfo;
import com.google.devtools.build.skyframe.EvaluationContext;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
import com.google.devtools.build.skyframe.MemoizingEvaluator;
import com.google.devtools.build.skyframe.RecordingDifferencer;
import com.google.devtools.build.skyframe.SequencedRecordingDifferencer;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
import net.starlark.java.eval.StarlarkSemantics;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/**
* Tests for {@link GlobFunction}.
*/
public abstract class GlobFunctionTest {
private static final EvaluationContext EVALUATION_OPTIONS =
EvaluationContext.newBuilder()
.setKeepGoing(false)
.setNumThreads(SkyframeExecutor.DEFAULT_THREAD_COUNT)
.setEventHandler(NullEventHandler.INSTANCE)
.build();
@RunWith(JUnit4.class)
public static class GlobFunctionAlwaysUseDirListingTest extends GlobFunctionTest {
@Override
protected boolean alwaysUseDirListing() {
return true;
}
}
@RunWith(JUnit4.class)
public static class RegularGlobFunctionTest extends GlobFunctionTest {
@Override
protected boolean alwaysUseDirListing() {
return false;
}
}
private CustomInMemoryFs fs;
private MemoizingEvaluator evaluator;
private RecordingDifferencer differencer;
private Path root;
private Path writableRoot;
private Path pkgPath;
private AtomicReference<PathPackageLocator> pkgLocator;
private static final PackageIdentifier PKG_ID = PackageIdentifier.createInMainRepo("pkg");
@Before
public final void setUp() throws Exception {
fs = new CustomInMemoryFs(new ManualClock());
root = fs.getPath("/root/workspace");
writableRoot = fs.getPath("/writableRoot/workspace");
pkgPath = root.getRelative(PKG_ID.getPackageFragment());
pkgLocator =
new AtomicReference<>(
new PathPackageLocator(
fs.getPath("/output_base"),
ImmutableList.of(Root.fromPath(writableRoot), Root.fromPath(root)),
BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY));
differencer = new SequencedRecordingDifferencer();
evaluator = new InMemoryMemoizingEvaluator(createFunctionMap(), differencer);
PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
PrecomputedValue.STARLARK_SEMANTICS.set(differencer, StarlarkSemantics.DEFAULT);
RepositoryDelegatorFunction.RESOLVED_FILE_INSTEAD_OF_WORKSPACE.set(
differencer, Optional.empty());
createTestFiles();
}
private Map<SkyFunctionName, SkyFunction> createFunctionMap() {
AtomicReference<ImmutableSet<PackageIdentifier>> deletedPackages =
new AtomicReference<>(ImmutableSet.of());
BlazeDirectories directories =
new BlazeDirectories(
new ServerDirectories(root, root, root),
root,
/* defaultSystemJavabase= */ null,
TestConstants.PRODUCT_NAME);
ExternalFilesHelper externalFilesHelper =
ExternalFilesHelper.createForTesting(
pkgLocator,
ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS,
directories);
Map<SkyFunctionName, SkyFunction> skyFunctions = new HashMap<>();
skyFunctions.put(SkyFunctions.GLOB, new GlobFunction(alwaysUseDirListing()));
skyFunctions.put(
SkyFunctions.DIRECTORY_LISTING_STATE,
new DirectoryListingStateFunction(externalFilesHelper, SyscallCache.NO_CACHE));
skyFunctions.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction());
skyFunctions.put(
SkyFunctions.PACKAGE_LOOKUP,
new PackageLookupFunction(
deletedPackages,
CrossRepositoryLabelViolationStrategy.ERROR,
BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY,
BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER));
skyFunctions.put(
SkyFunctions.IGNORED_PACKAGE_PREFIXES,
BazelSkyframeExecutorConstants.IGNORED_PACKAGE_PREFIXES_FUNCTION);
skyFunctions.put(
FileStateKey.FILE_STATE,
new FileStateFunction(
Suppliers.ofInstance(new TimestampGranularityMonitor(BlazeClock.instance())),
SyscallCache.NO_CACHE,
externalFilesHelper));
skyFunctions.put(FileValue.FILE, new FileFunction(pkgLocator, directories));
skyFunctions.put(
FileSymlinkCycleUniquenessFunction.NAME, new FileSymlinkCycleUniquenessFunction());
AnalysisMock analysisMock = AnalysisMock.get();
RuleClassProvider ruleClassProvider = analysisMock.createRuleClassProvider();
skyFunctions.put(
WorkspaceFileValue.WORKSPACE_FILE,
new WorkspaceFileFunction(
ruleClassProvider,
analysisMock
.getPackageFactoryBuilderForTesting(directories)
.build(ruleClassProvider, fs),
directories,
/*bzlLoadFunctionForInlining=*/ null));
skyFunctions.put(
SkyFunctions.EXTERNAL_PACKAGE,
new ExternalPackageFunction(BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER));
skyFunctions.put(
SkyFunctions.LOCAL_REPOSITORY_LOOKUP,
new LocalRepositoryLookupFunction(BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER));
return skyFunctions;
}
protected abstract boolean alwaysUseDirListing();
private void createTestFiles() throws IOException {
pkgPath.createDirectoryAndParents();
FileSystemUtils.createEmptyFile(pkgPath.getRelative("BUILD"));
for (String dir :
ImmutableList.of(
"foo/bar/wiz", "foo/barnacle/wiz", "food/barnacle/wiz", "fool/barnacle/wiz")) {
pkgPath.getRelative(dir).createDirectoryAndParents();
}
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/bar/wiz/file"));
// Used for testing the behavior of globbing into nested subpackages.
for (String dir : ImmutableList.of("a1/b1/c", "a2/b2/c")) {
pkgPath.getRelative(dir).createDirectoryAndParents();
}
FileSystemUtils.createEmptyFile(pkgPath.getRelative("a2/b2/BUILD"));
}
@Test
public void testSimple() throws Exception {
assertGlobMatches("food", /* => */ "food");
}
@Test
public void testIgnoreList() throws Exception {
FileSystemUtils.writeContentAsLatin1(root.getRelative(".bazelignore"), "pkg/foo/bar");
assertGlobMatches("foo/**", "foo/barnacle/wiz", "foo/barnacle", "foo");
differencer.invalidate(
ImmutableList.of(
FileStateValue.key(
RootedPath.toRootedPath(
Root.fromPath(root), PathFragment.create(".bazelignore")))));
FileSystemUtils.createEmptyFile(root.getRelative(".bazelignore"));
assertGlobMatches(
"foo/**",
"foo/bar/wiz",
"foo/bar/wiz/file",
"foo/bar",
"foo/barnacle/wiz",
"foo/barnacle",
"foo");
}
@Test
public void testStartsWithStar() throws Exception {
assertGlobMatches("*oo", /* => */ "foo");
}
@Test
public void testStartsWithStarWithMiddleStar() throws Exception {
assertGlobMatches("*f*o", /* => */ "foo");
}
@Test
public void testSingleMatchEqual() throws Exception {
assertGlobsEqual("*oo", "*f*o"); // both produce "foo"
}
@Test
public void testEndsWithStar() throws Exception {
assertGlobMatches("foo*", /* => */ "foo", "food", "fool");
}
@Test
public void testEndsWithStarWithMiddleStar() throws Exception {
assertGlobMatches("f*oo*", /* => */ "foo", "food", "fool");
}
@Test
public void testMultipleMatchesEqual() throws Exception {
assertGlobsEqual("foo*", "f*oo*"); // both produce "foo", "food", "fool"
}
@Test
public void testMiddleStar() throws Exception {
assertGlobMatches("f*o", /* => */ "foo");
}
@Test
public void testTwoMiddleStars() throws Exception {
assertGlobMatches("f*o*o", /* => */ "foo");
}
@Test
public void testSingleStarPatternWithNamedChild() throws Exception {
assertGlobMatches("*/bar", /* => */ "foo/bar");
}
@Test
public void testDeepSubpackages() throws Exception {
assertGlobMatches("*/*/c", /* => */ "a1/b1/c");
}
@Test
public void testSingleStarPatternWithChildGlob() throws Exception {
assertGlobMatches(
"*/bar*", /* => */ "foo/bar", "foo/barnacle", "food/barnacle", "fool/barnacle");
}
@Test
public void testSingleStarAsChildGlob() throws Exception {
assertGlobMatches("foo/*/wiz", /* => */ "foo/bar/wiz", "foo/barnacle/wiz");
}
@Test
public void testNoAsteriskAndFilesDontExist() throws Exception {
// Note un-UNIX like semantics:
assertGlobMatches("ceci/n'est/pas/une/globbe" /* => nothing */);
}
@Test
public void testSingleAsteriskUnderNonexistentDirectory() throws Exception {
// Note un-UNIX like semantics:
assertGlobMatches("not-there/*" /* => nothing */);
}
@Test
public void testDifferentGlobsSameResultEqual() throws Exception {
// Once the globs are run, it doesn't matter what pattern ran; only the output.
assertGlobsEqual("not-there/*", "syzygy/*"); // Both produce nothing.
}
@Test
public void testGlobUnderFile() throws Exception {
assertGlobMatches("foo/bar/wiz/file/*" /* => nothing */);
}
@Test
public void testGlobEqualsHashCode() throws Exception {
// Each "equality group" forms a set of elements that are all equals() to one another,
// and also produce the same hashCode.
new EqualsTester()
.addEqualityGroup(
runGlob("no-such-file", Globber.Operation.FILES_AND_DIRS)) // Matches nothing.
.addEqualityGroup(
runGlob("BUILD", Globber.Operation.FILES_AND_DIRS),
runGlob("BUILD", Globber.Operation.FILES)) // Matches BUILD.
.addEqualityGroup(
runGlob("**", Globber.Operation.FILES_AND_DIRS)) // Matches lots of things.
.addEqualityGroup(
runGlob("f*o/bar*", Globber.Operation.FILES_AND_DIRS),
runGlob(
"foo/bar*", Globber.Operation.FILES_AND_DIRS)) // Matches foo/bar and foo/barnacle.
.testEquals();
}
@Test
public void testGlobDoesNotCrossPackageBoundary() throws Exception {
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/BUILD"));
// "foo/bar" should not be in the results because foo is a separate package.
assertGlobMatches("f*/*", /* => */ "food/barnacle", "fool/barnacle");
}
@Test
public void testGlobDirectoryMatchDoesNotCrossPackageBoundary() throws Exception {
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/bar/BUILD"));
// "foo/bar" should not be in the results because foo/bar is a separate package.
assertGlobMatches("foo/*", /* => */ "foo/barnacle");
}
@Test
public void testStarStarDoesNotCrossPackageBoundary() throws Exception {
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/bar/BUILD"));
// "foo/bar" should not be in the results because foo/bar is a separate package.
assertGlobMatches("foo/**", /* => */ "foo/barnacle/wiz", "foo/barnacle", "foo");
}
@Test
public void testGlobDoesNotCrossPackageBoundaryUnderOtherPackagePath() throws Exception {
writableRoot.getRelative("pkg/foo/bar").createDirectoryAndParents();
FileSystemUtils.createEmptyFile(writableRoot.getRelative("pkg/foo/bar/BUILD"));
// "foo/bar" should not be in the results because foo/bar is detected as a separate package,
// even though it is under a different package path.
assertGlobMatches("foo/**", /* => */ "foo/barnacle/wiz", "foo/barnacle", "foo");
}
@Test
public void testGlobDoesNotCrossRepositoryBoundary() throws Exception {
FileSystemUtils.appendIsoLatin1(
root.getRelative("WORKSPACE"), "local_repository(name='local', path='pkg/foo')");
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/WORKSPACE"));
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/BUILD"));
// "foo/bar" should not be in the results because foo is a separate repository.
assertGlobMatches("f*/*", /* => */ "food/barnacle", "fool/barnacle");
}
@Test
public void testGlobDirectoryMatchDoesNotCrossRepositoryBoundary() throws Exception {
FileSystemUtils.appendIsoLatin1(
root.getRelative("WORKSPACE"), "local_repository(name='local', path='pkg/foo/bar')");
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/bar/WORKSPACE"));
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/bar/BUILD"));
// "foo/bar" should not be in the results because foo/bar is a separate repository.
assertGlobMatches("foo/*", /* => */ "foo/barnacle");
}
@Test
public void testStarStarDoesNotCrossRepositoryBoundary() throws Exception {
FileSystemUtils.appendIsoLatin1(
root.getRelative("WORKSPACE"), "local_repository(name='local', path='pkg/foo/bar')");
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/bar/WORKSPACE"));
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/bar/BUILD"));
// "foo/bar" should not be in the results because foo/bar is a separate repository.
assertGlobMatches("foo/**", /* => */ "foo/barnacle/wiz", "foo/barnacle", "foo");
}
@Test
public void testGlobDoesNotCrossRepositoryBoundaryUnderOtherPackagePath() throws Exception {
FileSystemUtils.appendIsoLatin1(
root.getRelative("WORKSPACE"),
"local_repository(name='local', path='"
+ writableRoot.getRelative("pkg/foo/bar").getPathString()
+ "')");
writableRoot.getRelative("pkg/foo/bar").createDirectoryAndParents();
FileSystemUtils.createEmptyFile(writableRoot.getRelative("pkg/foo/bar/WORKSPACE"));
FileSystemUtils.createEmptyFile(writableRoot.getRelative("pkg/foo/bar/BUILD"));
// "foo/bar" should not be in the results because foo/bar is detected as a separate package,
// even though it is under a different package path.
assertGlobMatches("foo/**", /* => */ "foo/barnacle/wiz", "foo/barnacle", "foo");
}
private void assertGlobMatches(String pattern, String... expecteds) throws Exception {
assertGlobMatches(pattern, Globber.Operation.FILES_AND_DIRS, expecteds);
}
private void assertGlobMatches(
String pattern, Globber.Operation globberOperation, String... expecteds) throws Exception {
// The order requirement is not strictly necessary -- a change to GlobFunction semantics that
// changes the output order is fine, but we require that the order be the same here to detect
// potential non-determinism in output order, which would be bad.
// The current order in the case of "**" or "*" is roughly that of nestedset.Order.STABLE_ORDER,
// putting subdirectories before directories, but putting ordinary files after their parent
// directories.
assertThat(
Iterables.transform(
runGlob(pattern, globberOperation).getMatches().toList(),
Functions.toStringFunction()))
.containsExactlyElementsIn(ImmutableList.copyOf(expecteds))
.inOrder();
}
private void assertGlobWithoutDirsMatches(String pattern, String... expecteds) throws Exception {
assertGlobMatches(pattern, Globber.Operation.FILES, expecteds);
}
private void assertGlobsEqual(String pattern1, String pattern2) throws Exception {
GlobValue value1 = runGlob(pattern1, Globber.Operation.FILES_AND_DIRS);
GlobValue value2 = runGlob(pattern2, Globber.Operation.FILES_AND_DIRS);
new EqualsTester()
.addEqualityGroup(value1, value2)
.testEquals();
}
private GlobValue runGlob(String pattern, Globber.Operation globberOperation) throws Exception {
SkyKey skyKey =
GlobValue.key(
PKG_ID, Root.fromPath(root), pattern, globberOperation, PathFragment.EMPTY_FRAGMENT);
EvaluationResult<SkyValue> result =
evaluator.evaluate(ImmutableList.of(skyKey), EVALUATION_OPTIONS);
if (result.hasError()) {
throw result.getError().getException();
}
return (GlobValue) result.get(skyKey);
}
@Test
public void testGlobWithoutWildcards() throws Exception {
String pattern = "foo/bar/wiz/file";
assertGlobMatches(pattern, "foo/bar/wiz/file");
// Ensure that the glob depends on the FileValue and not on the DirectoryListingValue.
pkgPath.getRelative("foo/bar/wiz/file").delete();
// Nothing has been invalidated yet, so the cached result is returned.
assertGlobMatches(pattern, "foo/bar/wiz/file");
if (alwaysUseDirListing()) {
differencer.invalidate(
ImmutableList.of(
FileStateValue.key(
RootedPath.toRootedPath(
Root.fromPath(root), pkgPath.getRelative("foo/bar/wiz/file")))));
// The result should not rely on the FileStateValue, so it's still a cache hit.
assertGlobMatches(pattern, "foo/bar/wiz/file");
differencer.invalidate(
ImmutableList.of(
DirectoryListingStateValue.key(
RootedPath.toRootedPath(
Root.fromPath(root), pkgPath.getRelative("foo/bar/wiz")))));
} else {
differencer.invalidate(
ImmutableList.of(
DirectoryListingStateValue.key(
RootedPath.toRootedPath(
Root.fromPath(root), pkgPath.getRelative("foo/bar/wiz")))));
// The result should not rely on the DirectoryListingValue, so it's still a cache hit.
assertGlobMatches(pattern, "foo/bar/wiz/file");
differencer.invalidate(
ImmutableList.of(
FileStateValue.key(
RootedPath.toRootedPath(
Root.fromPath(root), pkgPath.getRelative("foo/bar/wiz/file")))));
}
// This should have invalidated the glob result.
assertGlobMatches(pattern /* => nothing */);
}
@Test
public void testIllegalPatterns() throws Exception {
assertIllegalPattern("foo**bar");
assertIllegalPattern("?");
assertIllegalPattern("");
assertIllegalPattern(".");
assertIllegalPattern("/foo");
assertIllegalPattern("./foo");
assertIllegalPattern("foo/");
assertIllegalPattern("foo/./bar");
assertIllegalPattern("../foo/bar");
assertIllegalPattern("foo//bar");
}
@Test
public void testIllegalRecursivePatterns() throws Exception {
for (String prefix : Lists.newArrayList("", "*/", "**/", "ba/")) {
String suffix = ("/" + prefix).substring(0, prefix.length());
for (String pattern : Lists.newArrayList("**fo", "fo**", "**fo**", "fo**fo", "fo**fo**fo")) {
assertIllegalPattern(prefix + pattern);
assertIllegalPattern(pattern + suffix);
}
}
}
private void assertIllegalPattern(String pattern) {
assertThrows(
"invalid pattern not detected: " + pattern,
InvalidGlobPatternException.class,
() ->
GlobValue.key(
PKG_ID,
Root.fromPath(root),
pattern,
Globber.Operation.FILES_AND_DIRS,
PathFragment.EMPTY_FRAGMENT));
}
/**
* Tests that globs can contain Java regular expression special characters
*/
@Test
public void testSpecialRegexCharacter() throws Exception {
Path aDotB = pkgPath.getChild("a.b");
FileSystemUtils.createEmptyFile(aDotB);
FileSystemUtils.createEmptyFile(pkgPath.getChild("aab"));
// Note: this contains two asterisks because otherwise a RE is not built,
// as an optimization.
assertThat(
new UnixGlob.Builder(pkgPath, SyscallCache.NO_CACHE)
.addPattern("*a.b*")
.globInterruptible())
.containsExactly(aDotB);
}
@Test
public void testMatchesCallWithNoCache() {
assertThat(UnixGlob.matches("*a*b", "CaCb", null)).isTrue();
}
@Test
public void testHiddenFiles() throws Exception {
for (String dir : ImmutableList.of(".hidden", "..also.hidden", "not.hidden")) {
pkgPath.getRelative(dir).createDirectoryAndParents();
}
// Note that these are not in the result: ".", ".."
assertGlobMatches(
"*", "..also.hidden", ".hidden", "BUILD", "a1", "a2", "foo", "food", "fool", "not.hidden");
assertGlobMatches("*.hidden", "not.hidden");
}
@Test
public void testDoubleStar() throws Exception {
assertGlobMatches(
"**",
"a1/b1/c",
"a1/b1",
"a1",
"a2",
"foo/bar/wiz",
"foo/bar/wiz/file",
"foo/bar",
"foo/barnacle/wiz",
"foo/barnacle",
"foo",
"food/barnacle/wiz",
"food/barnacle",
"food",
"fool/barnacle/wiz",
"fool/barnacle",
"fool",
"BUILD");
}
@Test
public void testDoubleStarExcludeDirs() throws Exception {
assertGlobWithoutDirsMatches("**", "foo/bar/wiz/file", "BUILD");
}
@Test
public void testDoubleDoubleStar() throws Exception {
assertGlobMatches(
"**/**",
"a1/b1/c",
"a1/b1",
"a1",
"a2",
"foo/bar/wiz",
"foo/bar/wiz/file",
"foo/bar",
"foo/barnacle/wiz",
"foo/barnacle",
"foo",
"food/barnacle/wiz",
"food/barnacle",
"food",
"fool/barnacle/wiz",
"fool/barnacle",
"fool",
"BUILD");
}
@Test
public void testDirectoryWithDoubleStar() throws Exception {
assertGlobMatches(
"foo/**",
"foo/bar/wiz",
"foo/bar/wiz/file",
"foo/bar",
"foo/barnacle/wiz",
"foo/barnacle",
"foo");
}
@Test
public void testDoubleStarPatternWithNamedChild() throws Exception {
assertGlobMatches("**/bar", "foo/bar");
}
@Test
public void testDoubleStarPatternWithErrorChild() throws Exception {
FileSystemUtils.ensureSymbolicLink(pkgPath.getChild("self"), "self");
IOException ioException =
assertThrows(IOException.class, () -> runGlob("**/self", Operation.FILES));
assertThat(ioException).hasMessageThat().matches("Symlink cycle");
}
@Test
public void testDoubleStarPatternWithChildGlob() throws Exception {
assertGlobMatches("**/ba*", "foo/bar", "foo/barnacle", "food/barnacle", "fool/barnacle");
}
@Test
public void testDoubleStarAsChildGlob() throws Exception {
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/barnacle/wiz/wiz"));
pkgPath.getRelative("foo/barnacle/baz/wiz").createDirectoryAndParents();
assertGlobMatches(
"foo/**/wiz",
"foo/bar/wiz",
"foo/barnacle/wiz",
"foo/barnacle/baz/wiz",
"foo/barnacle/wiz/wiz");
}
@Test
public void testDoubleStarUnderNonexistentDirectory() throws Exception {
assertGlobMatches("not-there/**" /* => nothing */);
}
@Test
public void testDoubleStarUnderFile() throws Exception {
assertGlobMatches("foo/bar/wiz/file/**" /* => nothing */);
}
/** Regression test for b/225434889: Value with exception will not crash. */
@Test
public void symlinkFileValueWithError() throws Exception {
FileSystemUtils.ensureSymbolicLink(pkgPath.getChild("self"), "self");
IOException ioException =
assertThrows(IOException.class, () -> runGlob("self", Operation.FILES_AND_DIRS));
assertThat(ioException).hasMessageThat().matches("Symlink cycle");
}
@Test
public void symlinkSubdirValueWithError() throws Exception {
Path cycle = pkgPath.getChild("cycle");
FileSystemUtils.ensureSymbolicLink(cycle.getChild("self"), "self");
FileSystemUtils.ensureSymbolicLink(pkgPath.getChild("symlink"), cycle);
IOException ioException =
assertThrows(IOException.class, () -> runGlob("symlink/self", Operation.FILES_AND_DIRS));
assertThat(ioException).hasMessageThat().matches("Symlink cycle");
}
/** Regression test for b/13319874: Directory listing crash. */
@Test
public void testResilienceToFilesystemInconsistencies_directoryExistence() throws Exception {
// Our custom filesystem says "pkgPath/BUILD" exists but "pkgPath" does not exist.
fs.stubStat(pkgPath, null);
RootedPath pkgRootedPath = RootedPath.toRootedPath(Root.fromPath(root), pkgPath);
FileStateValue pkgDirFileStateValue =
FileStateValue.create(pkgRootedPath, SyscallCache.NO_CACHE, /*tsgm=*/ null);
FileValue pkgDirValue =
FileValue.value(
ImmutableList.of(pkgRootedPath),
null,
null,
pkgRootedPath,
pkgDirFileStateValue,
pkgRootedPath,
pkgDirFileStateValue);
differencer.inject(ImmutableMap.of(FileValue.key(pkgRootedPath), pkgDirValue));
String expectedMessage = "/root/workspace/pkg is no longer an existing directory";
SkyKey skyKey =
GlobValue.key(
PKG_ID,
Root.fromPath(root),
"*/foo",
Globber.Operation.FILES_AND_DIRS,
PathFragment.EMPTY_FRAGMENT);
EvaluationResult<GlobValue> result =
evaluator.evaluate(ImmutableList.of(skyKey), EVALUATION_OPTIONS);
assertThat(result.hasError()).isTrue();
ErrorInfo errorInfo = result.getError(skyKey);
assertThat(errorInfo.getException()).isInstanceOf(InconsistentFilesystemException.class);
assertThat(errorInfo.getException()).hasMessageThat().contains(expectedMessage);
}
@Test
public void testResilienceToFilesystemInconsistencies_subdirectoryExistence() throws Exception {
// Our custom filesystem says directory "pkgPath/foo/bar" contains a subdirectory "wiz" but a
// direct stat on "pkgPath/foo/bar/wiz" says it does not exist.
Path fooBarDir = pkgPath.getRelative("foo/bar");
fs.stubStat(fooBarDir.getRelative("wiz"), null);
RootedPath fooBarDirRootedPath = RootedPath.toRootedPath(Root.fromPath(root), fooBarDir);
SkyValue fooBarDirListingValue =
DirectoryListingStateValue.create(
ImmutableList.of(new Dirent("wiz", Dirent.Type.DIRECTORY)));
differencer.inject(
ImmutableMap.of(
DirectoryListingStateValue.key(fooBarDirRootedPath), fooBarDirListingValue));
String expectedMessage = "/root/workspace/pkg/foo/bar/wiz is no longer an existing directory.";
SkyKey skyKey =
GlobValue.key(
PKG_ID,
Root.fromPath(root),
"**/wiz",
Globber.Operation.FILES_AND_DIRS,
PathFragment.EMPTY_FRAGMENT);
EvaluationResult<GlobValue> result =
evaluator.evaluate(ImmutableList.of(skyKey), EVALUATION_OPTIONS);
assertThat(result.hasError()).isTrue();
ErrorInfo errorInfo = result.getError(skyKey);
assertThat(errorInfo.getException()).isInstanceOf(InconsistentFilesystemException.class);
assertThat(errorInfo.getException()).hasMessageThat().contains(expectedMessage);
}
@Test
public void testResilienceToFilesystemInconsistencies_symlinkType() throws Exception {
RootedPath wizRootedPath =
RootedPath.toRootedPath(Root.fromPath(root), pkgPath.getRelative("foo/bar/wiz"));
RootedPath fileRootedPath =
RootedPath.toRootedPath(Root.fromPath(root), pkgPath.getRelative("foo/bar/wiz/file"));
final FileStatus realStat = fileRootedPath.asPath().stat();
fs.stubStat(
fileRootedPath.asPath(),
new FileStatus() {
@Override
public boolean isFile() {
// The stat says foo/bar/wiz/file is a real file, not a symlink.
return true;
}
@Override
public boolean isSpecialFile() {
return false;
}
@Override
public boolean isDirectory() {
return false;
}
@Override
public boolean isSymbolicLink() {
return false;
}
@Override
public long getSize() throws IOException {
return realStat.getSize();
}
@Override
public long getLastModifiedTime() throws IOException {
return realStat.getLastModifiedTime();
}
@Override
public long getLastChangeTime() throws IOException {
return realStat.getLastChangeTime();
}
@Override
public long getNodeId() throws IOException {
return realStat.getNodeId();
}
});
// But the dir listing say foo/bar/wiz/file is a symlink.
SkyValue wizDirListingValue =
DirectoryListingStateValue.create(
ImmutableList.of(new Dirent("file", Dirent.Type.SYMLINK)));
differencer.inject(
ImmutableMap.of(DirectoryListingStateValue.key(wizRootedPath), wizDirListingValue));
String expectedMessage =
"readdir and stat disagree about whether " + fileRootedPath.asPath() + " is a symlink";
SkyKey skyKey =
GlobValue.key(
PKG_ID,
Root.fromPath(root),
"foo/bar/wiz/*",
Globber.Operation.FILES_AND_DIRS,
PathFragment.EMPTY_FRAGMENT);
EvaluationResult<GlobValue> result =
evaluator.evaluate(ImmutableList.of(skyKey), EVALUATION_OPTIONS);
assertThat(result.hasError()).isTrue();
ErrorInfo errorInfo = result.getError(skyKey);
assertThat(errorInfo.getException()).isInstanceOf(InconsistentFilesystemException.class);
assertThat(errorInfo.getException()).hasMessageThat().contains(expectedMessage);
}
@Test
public void testSymlinks() throws Exception {
pkgPath.getRelative("symlinks").createDirectoryAndParents();
FileSystemUtils.ensureSymbolicLink(pkgPath.getRelative("symlinks/dangling.txt"), "nope");
FileSystemUtils.createEmptyFile(pkgPath.getRelative("symlinks/yup"));
FileSystemUtils.ensureSymbolicLink(pkgPath.getRelative("symlinks/existing.txt"), "yup");
assertGlobMatches("symlinks/*.txt", "symlinks/existing.txt");
}
private static final class CustomInMemoryFs extends InMemoryFileSystem {
private final Map<PathFragment, FileStatus> stubbedStats = Maps.newHashMap();
CustomInMemoryFs(ManualClock manualClock) {
super(manualClock, DigestHashFunction.SHA256);
}
public void stubStat(Path path, @Nullable FileStatus stubbedResult) {
stubbedStats.put(path.asFragment(), stubbedResult);
}
@Override
public FileStatus statIfFound(PathFragment path, boolean followSymlinks) throws IOException {
if (stubbedStats.containsKey(path)) {
return stubbedStats.get(path);
}
return super.statIfFound(path, followSymlinks);
}
}
private void assertSubpackageMatches(String pattern, String... expecteds) throws Exception {
assertThat(
Iterables.transform(
runGlob(pattern, Globber.Operation.SUBPACKAGES).getMatches().toList(),
Functions.toStringFunction()))
.containsExactlyElementsIn(ImmutableList.copyOf(expecteds));
}
private void makeEmptyPackage(Path newPackagePath) throws Exception {
newPackagePath.createDirectoryAndParents();
FileSystemUtils.createEmptyFile(newPackagePath.getRelative("BUILD"));
}
private void makeEmptyPackage(String path) throws Exception {
makeEmptyPackage(pkgPath.getRelative(path));
}
@Test
public void subpackages_simple() throws Exception {
makeEmptyPackage("horse");
makeEmptyPackage("monkey");
makeEmptyPackage("horse/saddle");
// "horse/saddle" should not be in the results because horse/saddle is too deep. a2/b2 added by
// setup().
assertSubpackageMatches("**", /* => */ "a2/b2", "horse", "monkey");
}
@Test
public void subpackages_empty() throws Exception {
assertSubpackageMatches("foo/*");
assertSubpackageMatches("foo/**");
}
@Test
public void subpackages_oneLevelDeep() throws Exception {
makeEmptyPackage("base/sub");
makeEmptyPackage("base/sub2");
makeEmptyPackage("base/sub3");
assertSubpackageMatches("base/*", /* => */ "base/sub", "base/sub2", "base/sub3");
assertSubpackageMatches("base/**", /* => */ "base/sub", "base/sub2", "base/sub3");
}
@Test
public void subpackages_oneLevel_notDeepEnough() throws Exception {
makeEmptyPackage("base/sub/pkg");
makeEmptyPackage("base/sub2/pkg");
makeEmptyPackage("base/sub3/pkg");
// * doesn't go deep enough
assertSubpackageMatches("base/*");
// But if we go with ** it works fine.
assertSubpackageMatches("base/**", /* => */ "base/sub/pkg", "base/sub2/pkg", "base/sub3/pkg");
}
@Test
public void subpackages_deepRecurse() throws Exception {
makeEmptyPackage("base/sub/1");
makeEmptyPackage("base/sub/2");
makeEmptyPackage("base/sub2/3");
makeEmptyPackage("base/sub2/4");
makeEmptyPackage("base/sub3/5");
makeEmptyPackage("base/sub3/6");
FileSystemUtils.createEmptyFile(pkgPath.getRelative("foo/bar/BUILD"));
// "foo/bar" should not be in the results because foo/bar is a separate package.
assertSubpackageMatches(
"base/*/*",
"base/sub/1",
"base/sub/2",
"base/sub2/3",
"base/sub2/4",
"base/sub3/5",
"base/sub3/6");
assertSubpackageMatches(
"base/**",
"base/sub/1",
"base/sub/2",
"base/sub2/3",
"base/sub2/4",
"base/sub3/5",
"base/sub3/6");
}
@Test
public void subpackages_middleWidlcard() throws Exception {
makeEmptyPackage("base/sub1/same");
makeEmptyPackage("base/sub2/same");
makeEmptyPackage("base/sub3/same");
makeEmptyPackage("base/sub4/same");
makeEmptyPackage("base/sub5/same");
makeEmptyPackage("base/sub6/same");
assertSubpackageMatches(
"base/*/same",
"base/sub1/same",
"base/sub2/same",
"base/sub3/same",
"base/sub4/same",
"base/sub5/same",
"base/sub6/same");
assertSubpackageMatches(
"base/**/same",
"base/sub1/same",
"base/sub2/same",
"base/sub3/same",
"base/sub4/same",
"base/sub5/same",
"base/sub6/same");
}
@Test
public void subpackages_noWildcard() throws Exception {
makeEmptyPackage("sub1");
makeEmptyPackage("sub2");
makeEmptyPackage("sub3/deep");
makeEmptyPackage("sub4/deeper/deeper");
assertSubpackageMatches("sub");
assertSubpackageMatches("sub1", "sub1");
assertSubpackageMatches("sub2", "sub2");
assertSubpackageMatches("sub3/deep", "sub3/deep");
assertSubpackageMatches("sub4/deeper/deeper", "sub4/deeper/deeper");
}
@Test
public void subpackages_testSymlinks() throws Exception {
Path newPackagePath = pkgPath.getRelative("path/to/pkg");
makeEmptyPackage(newPackagePath);
pkgPath.getRelative("symlinks").createDirectoryAndParents();
FileSystemUtils.ensureSymbolicLink(pkgPath.getRelative("symlinks/deeplink"), newPackagePath);
FileSystemUtils.ensureSymbolicLink(pkgPath.getRelative("shallowlink"), newPackagePath);
assertSubpackageMatches("**", "a2/b2", "symlinks/deeplink", "path/to/pkg", "shallowlink");
assertSubpackageMatches("*", "shallowlink");
assertSubpackageMatches("symlinks/**", "symlinks/deeplink");
assertSubpackageMatches("symlinks/*", "symlinks/deeplink");
}
}