// Copyright 2015 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.skyframe;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.packages.ConstantRuleVisibility;
import com.google.devtools.build.lib.packages.NoSuchTargetException;
import com.google.devtools.build.lib.packages.Preprocessor;
import com.google.devtools.build.lib.packages.util.SubincludePreprocessor;
import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
import com.google.devtools.build.lib.skyframe.util.SkyframeExecutorTestUtils;
import com.google.devtools.build.lib.testutil.ManualClock;
import com.google.devtools.build.lib.testutil.TestUtils;
import com.google.devtools.build.lib.util.BlazeClock;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.Dirent;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.ModifiedFileSet;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import com.google.devtools.build.skyframe.ErrorInfo;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.build.skyframe.RecordingDifferencer;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.common.options.Options;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

/**
 * Unit tests of specific functionality of PackageFunction. Note that it's already tested
 * indirectly in several other places.
 */
@RunWith(JUnit4.class)
public class PackageFunctionTest extends BuildViewTestCase {

  private CustomInMemoryFs fs = new CustomInMemoryFs(new ManualClock());

  private void preparePackageLoading(Path... roots) {
    PackageCacheOptions packageCacheOptions = Options.getDefaults(PackageCacheOptions.class);
    packageCacheOptions.defaultVisibility = ConstantRuleVisibility.PUBLIC;
    packageCacheOptions.showLoadingProgress = true;
    packageCacheOptions.globbingThreads = 7;
    getSkyframeExecutor()
        .preparePackageLoading(
            new PathPackageLocator(outputBase, ImmutableList.copyOf(roots)),
            packageCacheOptions,
            "",
            UUID.randomUUID(),
            ImmutableMap.<String, String>of(),
            new TimestampGranularityMonitor(BlazeClock.instance()));
  }

  @Override
  protected Preprocessor.Factory.Supplier getPreprocessorFactorySupplier() {
    return new SubincludePreprocessor.FactorySupplier();
  }

  @Override
  protected FileSystem createFileSystem() {
    return fs;
  }

  private PackageValue validPackage(SkyKey skyKey) throws InterruptedException {
    EvaluationResult<PackageValue> result = SkyframeExecutorTestUtils.evaluate(
        getSkyframeExecutor(), skyKey, /*keepGoing=*/false, reporter);
    if (result.hasError()) {
      fail(result.getError(skyKey).getException().getMessage());
    }
    PackageValue value = result.get(skyKey);
    assertFalse(value.getPackage().containsErrors());
    return value;
  }

  @Test
  public void testValidPackage() throws Exception {
    scratch.file("pkg/BUILD");
    validPackage(PackageValue.key(PackageIdentifier.parse("@//pkg")));
  }

  @Test
  public void testInconsistentNewPackage() throws Exception {
    scratch.file("pkg/BUILD", "subinclude('//foo:sub')");
    scratch.file("foo/sub");

    preparePackageLoading(rootDirectory);

    SkyKey pkgLookupKey = PackageLookupValue.key(new PathFragment("foo"));
    EvaluationResult<PackageLookupValue> result = SkyframeExecutorTestUtils.evaluate(
        getSkyframeExecutor(), pkgLookupKey, /*keepGoing=*/false, reporter);
    assertFalse(result.hasError());
    assertFalse(result.get(pkgLookupKey).packageExists());

    scratch.file("foo/BUILD");

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
    result = SkyframeExecutorTestUtils.evaluate(getSkyframeExecutor(),
        skyKey, /*keepGoing=*/false, reporter);
    assertTrue(result.hasError());
    Throwable exception = result.getError(skyKey).getException();
    assertThat(exception.getMessage()).contains("Inconsistent filesystem operations");
    assertThat(exception.getMessage()).contains("Unexpected package");
  }

  @Test
  public void testInconsistentMissingPackage() throws Exception {
    reporter.removeHandler(failFastHandler);
    Path root1 = fs.getPath("/root1");
    scratch.file("/root1/WORKSPACE");
    scratch.file("/root1/foo/sub");
    scratch.file("/root1/pkg/BUILD", "subinclude('//foo:sub')");

    Path root2 = fs.getPath("/root2");
    scratch.file("/root2/foo/BUILD");
    scratch.file("/root2/foo/sub");

    preparePackageLoading(root1, root2);

    SkyKey pkgLookupKey = PackageLookupValue.key(PackageIdentifier.parse("@//foo"));
    EvaluationResult<PackageLookupValue> result = SkyframeExecutorTestUtils.evaluate(
        getSkyframeExecutor(), pkgLookupKey, /*keepGoing=*/false, reporter);
    assertFalse(result.hasError());
    assertEquals(root2, result.get(pkgLookupKey).getRoot());

    scratch.file("/root1/foo/BUILD");

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//pkg"));
    result = SkyframeExecutorTestUtils.evaluate(getSkyframeExecutor(),
        skyKey, /*keepGoing=*/false, reporter);
    assertTrue(result.hasError());
    Throwable exception = result.getError(skyKey).getException();
    System.out.println("exception: " + exception.getMessage());
    assertThat(exception.getMessage()).contains("Inconsistent filesystem operations");
    assertThat(exception.getMessage()).contains("Inconsistent package location");
  }

  @Test
  public void testPropagatesFilesystemInconsistencies() throws Exception {
    reporter.removeHandler(failFastHandler);
    RecordingDifferencer differencer = getSkyframeExecutor().getDifferencerForTesting();
    Path pkgRoot = getSkyframeExecutor().getPathEntries().get(0);
    Path fooBuildFile = scratch.file("foo/BUILD");
    Path fooDir = fooBuildFile.getParentDirectory();

    // Our custom filesystem says "foo/BUILD" exists but its parent "foo" is a file.
    FileStatus inconsistentParentFileStatus = new FileStatus() {
      @Override
      public boolean isFile() {
        return true;
      }

      @Override
      public boolean isDirectory() {
        return false;
      }

      @Override
      public boolean isSymbolicLink() {
        return false;
      }

      @Override
      public boolean isSpecialFile() {
        return false;
      }

      @Override
      public long getSize() throws IOException {
        return 0;
      }

      @Override
      public long getLastModifiedTime() throws IOException {
        return 0;
      }

      @Override
      public long getLastChangeTime() throws IOException {
        return 0;
      }

      @Override
      public long getNodeId() throws IOException {
        return 0;
      }
    };
    fs.stubStat(fooDir, inconsistentParentFileStatus);
    RootedPath pkgRootedPath = RootedPath.toRootedPath(pkgRoot, fooDir);
    SkyValue fooDirValue = FileStateValue.create(pkgRootedPath, tsgm);
    differencer.inject(ImmutableMap.of(FileStateValue.key(pkgRootedPath), fooDirValue));
    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    String expectedMessage = "/workspace/foo/BUILD exists but its parent path /workspace/foo isn't "
        + "an existing directory";
    EvaluationResult<PackageValue> result = SkyframeExecutorTestUtils.evaluate(
        getSkyframeExecutor(), skyKey, /*keepGoing=*/false, reporter);
    assertTrue(result.hasError());
    ErrorInfo errorInfo = result.getError(skyKey);
    String errorMessage = errorInfo.getException().getMessage();
    assertThat(errorMessage).contains("Inconsistent filesystem operations");
    assertThat(errorMessage).contains(expectedMessage);
  }

  @Test
  public void testPropagatesFilesystemInconsistencies_Globbing() throws Exception {
    reporter.removeHandler(failFastHandler);
    RecordingDifferencer differencer = getSkyframeExecutor().getDifferencerForTesting();
    Path pkgRoot = getSkyframeExecutor().getPathEntries().get(0);
    scratch.file("foo/BUILD",
        "subinclude('//a:a')",
        "sh_library(name = 'foo', srcs = glob(['bar/**/baz.sh']))");
    scratch.file("a/BUILD");
    scratch.file("a/a");
    Path bazFile = scratch.file("foo/bar/baz/baz.sh");
    Path bazDir = bazFile.getParentDirectory();
    Path barDir = bazDir.getParentDirectory();

    // Our custom filesystem says "foo/bar/baz" does not exist but it also says that "foo/bar"
    // has a child directory "baz".
    fs.stubStat(bazDir, null);
    RootedPath barDirRootedPath = RootedPath.toRootedPath(pkgRoot, barDir);
    FileStateValue barDirFileStateValue = FileStateValue.create(barDirRootedPath, tsgm);
    FileValue barDirFileValue = FileValue.value(barDirRootedPath, barDirFileStateValue,
        barDirRootedPath, barDirFileStateValue);
    DirectoryListingValue barDirListing = DirectoryListingValue.value(barDirRootedPath,
        barDirFileValue, DirectoryListingStateValue.create(ImmutableList.of(
            new Dirent("baz", Dirent.Type.DIRECTORY))));
    differencer.inject(ImmutableMap.of(DirectoryListingValue.key(barDirRootedPath), barDirListing));
    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    String expectedMessage = "/workspace/foo/bar/baz is no longer an existing directory";
    EvaluationResult<PackageValue> result = SkyframeExecutorTestUtils.evaluate(
        getSkyframeExecutor(), skyKey, /*keepGoing=*/false, reporter);
    assertTrue(result.hasError());
    ErrorInfo errorInfo = result.getError(skyKey);
    String errorMessage = errorInfo.getException().getMessage();
    assertThat(errorMessage).contains("Inconsistent filesystem operations");
    assertThat(errorMessage).contains(expectedMessage);
  }

  /** Regression test for unexpected exception type from PackageValue. */
  @Test
  public void testDiscrepancyBetweenLegacyAndSkyframePackageLoadingErrors() throws Exception {
    reporter.removeHandler(failFastHandler);
    Path fooBuildFile = scratch.file("foo/BUILD",
        "sh_library(name = 'foo', srcs = glob(['bar/*.sh']))");
    Path fooDir = fooBuildFile.getParentDirectory();
    Path barDir = fooDir.getRelative("bar");
    scratch.file("foo/bar/baz.sh");
    fs.scheduleMakeUnreadableAfterReaddir(barDir);

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    String expectedMessage = "Encountered error 'Directory is not readable'";
    EvaluationResult<PackageValue> result = SkyframeExecutorTestUtils.evaluate(
        getSkyframeExecutor(), skyKey, /*keepGoing=*/false, reporter);
    assertTrue(result.hasError());
    ErrorInfo errorInfo = result.getError(skyKey);
    String errorMessage = errorInfo.getException().getMessage();
    assertThat(errorMessage).contains("Inconsistent filesystem operations");
    assertThat(errorMessage).contains(expectedMessage);
  }

  @Test
  public void testMultipleSubincludesFromSamePackage() throws Exception {
    scratch.file("foo/BUILD",
        "subinclude('//bar:a')",
        "subinclude('//bar:b')");
    scratch.file("bar/BUILD",
        "exports_files(['a', 'b'])");
    scratch.file("bar/a");
    scratch.file("bar/b");

    preparePackageLoading(rootDirectory);

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    validPackage(skyKey);
  }

  @Test
  public void testTransitiveSubincludesStoredInPackage() throws Exception {
    scratch.file("foo/BUILD",
        "subinclude('//bar:a')");
    scratch.file("bar/BUILD",
        "exports_files(['a'])");
    scratch.file("bar/a",
        "subinclude('//baz:b')");
    scratch.file("baz/BUILD",
        "exports_files(['b', 'c'])");
    scratch.file("baz/b");
    scratch.file("baz/c");

    preparePackageLoading(rootDirectory);

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    PackageValue value = validPackage(skyKey);
    assertThat(value.getPackage().getSubincludeLabels()).containsExactly(
        Label.parseAbsolute("//bar:a"), Label.parseAbsolute("//baz:b"));

    scratch.overwriteFile("bar/a",
        "subinclude('//baz:c')");
    getSkyframeExecutor().invalidateFilesUnderPathForTesting(reporter,
        ModifiedFileSet.builder().modify(new PathFragment("bar/a")).build(), rootDirectory);

    value = validPackage(skyKey);
    assertThat(value.getPackage().getSubincludeLabels()).containsExactly(
        Label.parseAbsolute("//bar:a"), Label.parseAbsolute("//baz:c"));
  }

  @SuppressWarnings("unchecked") // Cast of srcs attribute to Iterable<Label>.
  @Test
  public void testGlobOrderStable() throws Exception {
    scratch.file("foo/BUILD", "sh_library(name = 'foo', srcs = glob(['**/*.txt']))");
    scratch.file("foo/b.txt");
    scratch.file("foo/c/c.txt");
    preparePackageLoading(rootDirectory);
    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    PackageValue value = validPackage(skyKey);
    assertThat(
            (Iterable<Label>)
                value
                    .getPackage()
                    .getTarget("foo")
                    .getAssociatedRule()
                    .getAttributeContainer()
                    .getAttr("srcs"))
        .containsExactly(
            Label.parseAbsoluteUnchecked("//foo:b.txt"),
            Label.parseAbsoluteUnchecked("//foo:c/c.txt"))
        .inOrder();
    scratch.file("foo/d.txt");
    getSkyframeExecutor()
        .invalidateFilesUnderPathForTesting(
            reporter,
            ModifiedFileSet.builder().modify(new PathFragment("foo/d.txt")).build(),
            rootDirectory);
    value = validPackage(skyKey);
    assertThat(
            (Iterable<Label>)
                value
                    .getPackage()
                    .getTarget("foo")
                    .getAssociatedRule()
                    .getAttributeContainer()
                    .getAttr("srcs"))
        .containsExactly(
            Label.parseAbsoluteUnchecked("//foo:b.txt"),
            Label.parseAbsoluteUnchecked("//foo:c/c.txt"),
            Label.parseAbsoluteUnchecked("//foo:d.txt"))
        .inOrder();
  }

  @SuppressWarnings("unchecked") // Cast of srcs attribute to Iterable<Label>.
  @Test
  public void testGlobOrderStableWithLegacyAndSkyframeComponents() throws Exception {
    scratch.file("foo/BUILD", "sh_library(name = 'foo', srcs = glob(['*.txt']))");
    scratch.file("foo/b.txt");
    scratch.file("foo/a.config");
    preparePackageLoading(rootDirectory);
    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    PackageValue value = validPackage(skyKey);
    assertThat(
            (Iterable<Label>)
                value
                    .getPackage()
                    .getTarget("foo")
                    .getAssociatedRule()
                    .getAttributeContainer()
                    .getAttr("srcs"))
        .containsExactly(Label.parseAbsoluteUnchecked("//foo:b.txt"));
    scratch.overwriteFile(
        "foo/BUILD", "sh_library(name = 'foo', srcs = glob(['*.txt', '*.config']))");
    getSkyframeExecutor()
        .invalidateFilesUnderPathForTesting(
            reporter,
            ModifiedFileSet.builder().modify(new PathFragment("foo/BUILD")).build(),
            rootDirectory);
    value = validPackage(skyKey);
    assertThat(
            (Iterable<Label>)
                value
                    .getPackage()
                    .getTarget("foo")
                    .getAssociatedRule()
                    .getAttributeContainer()
                    .getAttr("srcs"))
        .containsExactly(
            Label.parseAbsoluteUnchecked("//foo:a.config"),
            Label.parseAbsoluteUnchecked("//foo:b.txt"))
        .inOrder();
    scratch.overwriteFile(
        "foo/BUILD", "sh_library(name = 'foo', srcs = glob(['*.txt', '*.config'])) # comment");
    getSkyframeExecutor()
        .invalidateFilesUnderPathForTesting(
            reporter,
            ModifiedFileSet.builder().modify(new PathFragment("foo/BUILD")).build(),
            rootDirectory);
    value = validPackage(skyKey);
    assertThat(
            (Iterable<Label>)
                value
                    .getPackage()
                    .getTarget("foo")
                    .getAssociatedRule()
                    .getAttributeContainer()
                    .getAttr("srcs"))
        .containsExactly(
            Label.parseAbsoluteUnchecked("//foo:a.config"),
            Label.parseAbsoluteUnchecked("//foo:b.txt"))
        .inOrder();
    getSkyframeExecutor().resetEvaluator();
    PackageCacheOptions packageCacheOptions = Options.getDefaults(PackageCacheOptions.class);
    packageCacheOptions.defaultVisibility = ConstantRuleVisibility.PUBLIC;
    packageCacheOptions.showLoadingProgress = true;
    packageCacheOptions.globbingThreads = 7;
    getSkyframeExecutor()
        .preparePackageLoading(
            new PathPackageLocator(outputBase, ImmutableList.<Path>of(rootDirectory)),
            packageCacheOptions,
            "",
            UUID.randomUUID(),
            ImmutableMap.<String, String>of(),
            tsgm);
    value = validPackage(skyKey);
    assertThat(
            (Iterable<Label>)
                value
                    .getPackage()
                    .getTarget("foo")
                    .getAssociatedRule()
                    .getAttributeContainer()
                    .getAttr("srcs"))
        .containsExactly(
            Label.parseAbsoluteUnchecked("//foo:a.config"),
            Label.parseAbsoluteUnchecked("//foo:b.txt"))
        .inOrder();
  }

  @Test
  public void testIncludeInMainAndDefaultRepository() throws Exception {
    scratch.file("foo/BUILD",
        "subinclude('//baz:a')");
    scratch.file("bar/BUILD",
        "subinclude('@//baz:a')");
    scratch.file("baz/BUILD",
        "exports_files(['a'])");
    scratch.file("baz/a");

    preparePackageLoading(rootDirectory);

    SkyKey fooKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    PackageValue fooValue = validPackage(fooKey);
    assertThat(fooValue.getPackage().getSubincludeLabels()).containsExactly(
        Label.parseAbsolute("//baz:a"));

    SkyKey barKey = PackageValue.key(PackageIdentifier.parse("@//bar"));
    PackageValue barValue = validPackage(barKey);
    assertThat(barValue.getPackage().getSubincludeLabels()).containsExactly(
        Label.parseAbsolute("@//baz:a"));
  }

  @Test
  public void testTransitiveSkylarkDepsStoredInPackage() throws Exception {
    scratch.file("foo/BUILD",
        "load('/bar/ext', 'a')");
    scratch.file("bar/BUILD");
    scratch.file("bar/ext.bzl",
        "load('/baz/ext', 'b')",
        "a = b");
    scratch.file("baz/BUILD");
    scratch.file("baz/ext.bzl",
        "b = 1");
    scratch.file("qux/BUILD");
    scratch.file("qux/ext.bzl",
        "c = 1");

    preparePackageLoading(rootDirectory);

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    PackageValue value = validPackage(skyKey);
    assertThat(value.getPackage().getSkylarkFileDependencies()).containsExactly(
        Label.parseAbsolute("//bar:ext.bzl"), Label.parseAbsolute("//baz:ext.bzl"));

    scratch.overwriteFile("bar/ext.bzl",
        "load('/qux/ext', 'c')",
        "a = c");
    getSkyframeExecutor().invalidateFilesUnderPathForTesting(reporter,
        ModifiedFileSet.builder().modify(new PathFragment("bar/ext.bzl")).build(), rootDirectory);

    value = validPackage(skyKey);
    assertThat(value.getPackage().getSkylarkFileDependencies()).containsExactly(
        Label.parseAbsolute("//bar:ext.bzl"), Label.parseAbsolute("//qux:ext.bzl"));
  }

  @Test
  public void testNonExistingSkylarkExtension() throws Exception {
    reporter.removeHandler(failFastHandler);
    scratch.file("test/skylark/BUILD",
        "load('/test/skylark/bad_extension', 'some_symbol')",
        "genrule(name = gr,",
        "    outs = ['out.txt'],",
        "    cmd = 'echo hello >@')");
    invalidatePackages();

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//test/skylark"));
    EvaluationResult<PackageValue> result = SkyframeExecutorTestUtils.evaluate(
        getSkyframeExecutor(), skyKey, /*keepGoing=*/false, reporter);
    assertTrue(result.hasError());
    ErrorInfo errorInfo = result.getError(skyKey);
    String expectedMsg = "error loading package 'test/skylark': "
        + "Extension file not found. Unable to load file '//test/skylark:bad_extension.bzl': "
        + "file doesn't exist or isn't a file";
    assertThat(errorInfo.getException())
        .hasMessage(expectedMsg);
  }

  @Test
  public void testNonExistingSkylarkExtensionWithPythonPreprocessing() throws Exception {
    reporter.removeHandler(failFastHandler);
    scratch.file("foo/BUILD",
        "exports_files(['a'])");
    scratch.file("foo/a",
        "load('/test/skylark/bad_extension', 'some_symbol')");
    scratch.file("test/skylark/BUILD",
        "subinclude('//foo:a')");
    invalidatePackages();

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//test/skylark"));
    EvaluationResult<PackageValue> result = SkyframeExecutorTestUtils.evaluate(
        getSkyframeExecutor(), skyKey, /*keepGoing=*/false, reporter);
    assertTrue(result.hasError());
    ErrorInfo errorInfo = result.getError(skyKey);
    String expectedMsg = "error loading package 'test/skylark': "
        + "Extension file not found. Unable to load file '//test/skylark:bad_extension.bzl': "
        + "file doesn't exist or isn't a file";
    assertThat(errorInfo.getException())
        .hasMessage(expectedMsg);
  }

  @Test
  public void testNonExistingSkylarkExtensionFromExtension() throws Exception {
    reporter.removeHandler(failFastHandler);
    scratch.file("test/skylark/extension.bzl",
        "load('/test/skylark/bad_extension', 'some_symbol')",
        "a = 'a'");
    scratch.file("test/skylark/BUILD",
        "load('/test/skylark/extension', 'a')",
        "genrule(name = gr,",
        "    outs = ['out.txt'],",
        "    cmd = 'echo hello >@')");
    invalidatePackages();

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//test/skylark"));
    EvaluationResult<PackageValue> result = SkyframeExecutorTestUtils.evaluate(
        getSkyframeExecutor(), skyKey, /*keepGoing=*/false, reporter);
    assertTrue(result.hasError());
    ErrorInfo errorInfo = result.getError(skyKey);
    assertThat(errorInfo.getException())
        .hasMessage("error loading package 'test/skylark': Extension file not found. "
            + "Unable to load file '//test/skylark:bad_extension.bzl': "
            + "file doesn't exist or isn't a file");
  }

  @Test
  public void testSymlinkCycleWithSkylarkExtension() throws Exception {
    reporter.removeHandler(failFastHandler);
    Path extensionFilePath = scratch.resolve("/workspace/test/skylark/extension.bzl");
    FileSystemUtils.ensureSymbolicLink(extensionFilePath, new PathFragment("extension.bzl"));
    scratch.file("test/skylark/BUILD",
        "load('/test/skylark/extension', 'a')",
        "genrule(name = gr,",
        "    outs = ['out.txt'],",
        "    cmd = 'echo hello >@')");
    invalidatePackages();

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//test/skylark"));
    EvaluationResult<PackageValue> result = SkyframeExecutorTestUtils.evaluate(
        getSkyframeExecutor(), skyKey, /*keepGoing=*/false, reporter);
    assertTrue(result.hasError());
    ErrorInfo errorInfo = result.getError(skyKey);
    assertEquals(skyKey, errorInfo.getRootCauseOfException());
    assertThat(errorInfo.getException())
        .hasMessage(
            "error loading package 'test/skylark': Encountered error while reading extension "
            + "file 'test/skylark/extension.bzl': Symlink cycle");
  }

  @Test
  public void testIOErrorLookingForSubpackageForLabelIsHandled() throws Exception {
    reporter.removeHandler(failFastHandler);
    scratch.file("foo/BUILD",
        "sh_library(name = 'foo', srcs = ['bar/baz.sh'])");
    Path barBuildFile = scratch.file("foo/bar/BUILD");
    fs.stubStatError(barBuildFile, new IOException("nope"));
    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    EvaluationResult<PackageValue> result = SkyframeExecutorTestUtils.evaluate(
        getSkyframeExecutor(), skyKey, /*keepGoing=*/false, reporter);
    assertTrue(result.hasError());
    assertContainsEvent("nope");
  }

  @Test
  public void testLoadRelativePath() throws Exception {
    scratch.file("pkg/BUILD", "load('ext', 'a')");
    scratch.file("pkg/ext.bzl", "a = 1");
    validPackage(PackageValue.key(PackageIdentifier.parse("@//pkg")));
  }

  @Test
  public void testLoadAbsolutePath() throws Exception {
    scratch.file("pkg1/BUILD");
    scratch.file("pkg2/BUILD",
        "load('/pkg1/ext', 'a')");
    scratch.file("pkg1/ext.bzl", "a = 1");
    validPackage(PackageValue.key(PackageIdentifier.parse("@//pkg2")));
  }

  @Test
  public void testBadWorkspaceFile() throws Exception {
    Path workspacePath = scratch.overwriteFile("WORKSPACE", "junk");
    SkyKey skyKey = PackageValue.key(PackageIdentifier.createInMainRepo("external"));
    getSkyframeExecutor()
        .invalidate(
            Predicates.equalTo(
                FileStateValue.key(
                    RootedPath.toRootedPath(
                        workspacePath.getParentDirectory(),
                        new PathFragment(workspacePath.getBaseName())))));

    reporter.removeHandler(failFastHandler);
    EvaluationResult<PackageValue> result =
        SkyframeExecutorTestUtils.evaluate(
            getSkyframeExecutor(), skyKey, /*keepGoing=*/ false, reporter);
    assertFalse(result.hasError());
    assertTrue(result.get(skyKey).getPackage().containsErrors());
  }

  // Regression test for the two ugly consequences of a bug where GlobFunction incorrectly matched
  // dangling symlinks.
  @Test
  public void testIncrementalSkyframeHybridGlobbingOnDanglingSymlink() throws Exception {
    Path packageDirPath = scratch.file("foo/BUILD",
        "exports_files(glob(['*.txt']))").getParentDirectory();
    scratch.file("foo/existing.txt");
    FileSystemUtils.ensureSymbolicLink(packageDirPath.getChild("dangling.txt"), "nope");

    preparePackageLoading(rootDirectory);

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    PackageValue value = validPackage(skyKey);
    assertFalse(value.getPackage().containsErrors());
    assertThat(value.getPackage().getTarget("existing.txt").getName()).isEqualTo("existing.txt");
    try {
      value.getPackage().getTarget("dangling.txt");
      fail();
    } catch (NoSuchTargetException expected) {
    }

    scratch.overwriteFile("foo/BUILD",
        "exports_files(glob(['*.txt'])),",
        "#some-irrelevant-comment");

    getSkyframeExecutor().invalidateFilesUnderPathForTesting(reporter,
        ModifiedFileSet.builder().modify(new PathFragment("foo/BUILD")).build(), rootDirectory);

    value = validPackage(skyKey);
    assertFalse(value.getPackage().containsErrors());
    assertThat(value.getPackage().getTarget("existing.txt").getName()).isEqualTo("existing.txt");
    try {
      value.getPackage().getTarget("dangling.txt");
      fail();
    } catch (NoSuchTargetException expected) {
      // One consequence of the bug was that dangling symlinks were matched by globs evaluated by
      // Skyframe globbing, meaning there would incorrectly be corresponding targets in packages
      // that had skyframe cache hits during skyframe hybrid globbing.
    }

    scratch.file("foo/nope");
    getSkyframeExecutor().invalidateFilesUnderPathForTesting(reporter,
        ModifiedFileSet.builder().modify(new PathFragment("foo/nope")).build(), rootDirectory);

    PackageValue newValue = validPackage(skyKey);
    assertFalse(newValue.getPackage().containsErrors());
    assertThat(newValue.getPackage().getTarget("existing.txt").getName()).isEqualTo("existing.txt");
    // Another consequence of the bug is that change pruning would incorrectly cut off changes that
    // caused a dangling symlink potentially matched by a glob to come into existence.
    assertThat(newValue.getPackage().getTarget("dangling.txt").getName()).isEqualTo("dangling.txt");
    assertThat(newValue.getPackage()).isNotSameAs(value.getPackage());
  }

  // Regression test for Skyframe globbing incorrectly matching the package's directory path on
  // 'glob(['**'], exclude_directories = 0)'. We test for this directly by triggering
  // hybrid globbing (gives coverage for both legacy globbing and skyframe globbing).
  @Test
  public void testRecursiveGlobNeverMatchesPackageDirectory() throws Exception {
    scratch.file("foo/BUILD",
        "[sh_library(name = x + '-matched') for x in glob(['**'], exclude_directories = 0)]");
    scratch.file("foo/bar");

    preparePackageLoading(rootDirectory);

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    PackageValue value = validPackage(skyKey);
    assertFalse(value.getPackage().containsErrors());
    assertThat(value.getPackage().getTarget("bar-matched").getName()).isEqualTo("bar-matched");
    try {
      value.getPackage().getTarget("-matched");
      fail();
    } catch (NoSuchTargetException expected) {
    }

    scratch.overwriteFile("foo/BUILD",
        "[sh_library(name = x + '-matched') for x in glob(['**'], exclude_directories = 0)]",
        "#some-irrelevant-comment");
    getSkyframeExecutor().invalidateFilesUnderPathForTesting(reporter,
        ModifiedFileSet.builder().modify(new PathFragment("foo/BUILD")).build(), rootDirectory);

    value = validPackage(skyKey);
    assertFalse(value.getPackage().containsErrors());
    assertThat(value.getPackage().getTarget("bar-matched").getName()).isEqualTo("bar-matched");
    try {
      value.getPackage().getTarget("-matched");
      fail();
    } catch (NoSuchTargetException expected) {
    }
  }

  @Test
  public void testGlobsHappenInParallel() throws Exception {
    scratch.file(
        "foo/BUILD",
        "load('//foo:my_library.bzl', 'my_library')",
        "[sh_library(name = x + '-matched') for x in glob(['bar/*'], exclude_directories = 0)]",
        "cc_library(name = 'cc', srcs = glob(['cc/*']))",
        "my_library(name = 'my', srcs = glob(['sh/*']))");
    scratch.file(
        "foo/my_library.bzl",
        "def my_library(name = None, srcs = []):",
        "  native.sh_library(name = name, srcs = srcs, deps = native.glob(['inner/*']))");
    scratch.file("foo/bar/1");
    Path barPath = scratch.file("foo/bar/2").getParentDirectory();
    Path ccPath = scratch.file("foo/cc/src.file").getParentDirectory();
    Path shPath = scratch.dir("foo/sh");
    Path innerPath = scratch.dir("foo/inner");
    PackageCacheOptions packageCacheOptions = Options.getDefaults(PackageCacheOptions.class);
    packageCacheOptions.defaultVisibility = ConstantRuleVisibility.PUBLIC;
    packageCacheOptions.showLoadingProgress = true;
    packageCacheOptions.globbingThreads = 7;
    packageCacheOptions.maxDirectoriesToEagerlyVisitInGlobbing = 10;
    getSkyframeExecutor()
        .preparePackageLoading(
            new PathPackageLocator(outputBase, ImmutableList.of(rootDirectory)),
            packageCacheOptions,
            "",
            UUID.randomUUID(),
            ImmutableMap.<String, String>of(),
            new TimestampGranularityMonitor(BlazeClock.instance()));

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    final CountDownLatch allDirsRequested = new CountDownLatch(4);
    Listener synchronizeListener =
        new Listener() {
          @Override
          public Object accept(Path path, FileOp op, Order order) throws IOException {
            if (op == FileOp.READDIR && order == Order.BEFORE) {
              allDirsRequested.countDown();
              try {
                assertThat(
                        allDirsRequested.await(
                            TestUtils.WAIT_TIMEOUT_MILLISECONDS, TimeUnit.MILLISECONDS))
                    .isTrue();
              } catch (InterruptedException e) {
                throw new IllegalStateException(e);
              }
            }
            return NO_RESULT_MARKER;
          }
        };
    fs.setCustomOverride(barPath, synchronizeListener);
    fs.setCustomOverride(ccPath, synchronizeListener);
    fs.setCustomOverride(shPath, synchronizeListener);
    fs.setCustomOverride(innerPath, synchronizeListener);
    PackageValue value = validPackage(skyKey);
    assertFalse(value.getPackage().containsErrors());
    assertThat(value.getPackage().getTarget("bar/1-matched").getName()).isEqualTo("bar/1-matched");
    assertThat(value.getPackage().getTarget("cc/src.file")).isNotNull();
    assertThat(
            (Iterable<?>)
                value
                    .getPackage()
                    .getTarget("my")
                    .getAssociatedRule()
                    .getAttributeContainer()
                    .getAttr("srcs"))
        .isEmpty();
  }

  @Test
  public void testGlobsDontHappenInParallel() throws Exception {
    scratch.file(
        "foo/BUILD",
        "load('//foo:my_library.bzl', 'my_library')",
        "[sh_library(name = x + '-matched') for x in glob(['bar/*'], exclude_directories = 0)]",
        "cc_library(name = 'cc', srcs = glob(['cc/*']))",
        "my_library(name = 'my', srcs = glob(['sh/*']))");
    scratch.file(
        "foo/my_library.bzl",
        "def my_library(name = None, srcs = []):",
        "  native.sh_library(name = name, srcs = srcs, deps = native.glob(['inner/*']))");
    scratch.file("foo/bar/1");
    Path barPath = scratch.file("foo/bar/2").getParentDirectory();
    Path ccPath = scratch.file("foo/cc/src.file").getParentDirectory();
    Path shPath = scratch.dir("foo/sh");
    Path innerPath = scratch.dir("foo/inner");
    PackageCacheOptions packageCacheOptions = Options.getDefaults(PackageCacheOptions.class);
    packageCacheOptions.defaultVisibility = ConstantRuleVisibility.PUBLIC;
    packageCacheOptions.showLoadingProgress = true;
    packageCacheOptions.globbingThreads = 7;
    packageCacheOptions.maxDirectoriesToEagerlyVisitInGlobbing = -1;
    getSkyframeExecutor()
        .preparePackageLoading(
            new PathPackageLocator(outputBase, ImmutableList.of(rootDirectory)),
            packageCacheOptions,
            "",
            UUID.randomUUID(),
            ImmutableMap.<String, String>of(),
            new TimestampGranularityMonitor(BlazeClock.instance()));

    SkyKey skyKey = PackageValue.key(PackageIdentifier.parse("@//foo"));
    final AtomicBoolean atLeastOneUnfinishedRequest = new AtomicBoolean(false);
    final CountDownLatch allDirsRequested = new CountDownLatch(4);
    Listener synchronizeListener =
        new Listener() {
          @Override
          public Object accept(Path path, FileOp op, Order order) throws IOException {
            if (op == FileOp.READDIR && order == Order.BEFORE) {
              allDirsRequested.countDown();
              try {
                if (!allDirsRequested.await(1, TimeUnit.SECONDS)) {
                  atLeastOneUnfinishedRequest.set(true);
                }
              } catch (InterruptedException e) {
                throw new IllegalStateException(e);
              }
            }
            return NO_RESULT_MARKER;
          }
        };
    fs.setCustomOverride(barPath, synchronizeListener);
    fs.setCustomOverride(ccPath, synchronizeListener);
    fs.setCustomOverride(shPath, synchronizeListener);
    fs.setCustomOverride(innerPath, synchronizeListener);
    PackageValue value = validPackage(skyKey);
    assertFalse(value.getPackage().containsErrors());
    assertThat(value.getPackage().getTarget("bar/1-matched").getName()).isEqualTo("bar/1-matched");
    assertThat(value.getPackage().getTarget("cc/src.file")).isNotNull();
    assertThat(
            (Iterable<?>)
                value
                    .getPackage()
                    .getTarget("my")
                    .getAssociatedRule()
                    .getAttributeContainer()
                    .getAttr("srcs"))
        .isEmpty();
    assertThat(atLeastOneUnfinishedRequest.get()).isTrue();
  }

  private static class CustomInMemoryFs extends InMemoryFileSystem {
    private final Map<Path, Listener> customOverrides = Maps.newHashMap();

    public CustomInMemoryFs(ManualClock manualClock) {
      super(manualClock);
    }

    public void stubStat(final Path targetPath, @Nullable final FileStatus stubbedResult) {
      setCustomOverride(
          targetPath,
          new Listener() {
            @Override
            public Object accept(Path path, FileOp op, Order order) {
              if (targetPath.equals(path) && op == FileOp.STAT && order == Order.BEFORE) {
                return stubbedResult;
              } else {
                return NO_RESULT_MARKER;
              }
            }
          });
    }

    public void stubStatError(final Path targetPath, final IOException stubbedResult) {
      setCustomOverride(
          targetPath,
          new Listener() {
            @Override
            public Object accept(Path path, FileOp op, Order order) throws IOException {
              if (targetPath.equals(path) && op == FileOp.STAT && order == Order.BEFORE) {
                throw stubbedResult;
              } else {
                return NO_RESULT_MARKER;
              }
            }
          });
    }

    void setCustomOverride(Path path, Listener listener) {
      customOverrides.put(path, listener);
    }

    @Override
    public FileStatus stat(Path path, boolean followSymlinks) throws IOException {
      Listener listener = customOverrides.get(path);
      if (listener != null) {
        Object status = listener.accept(path, FileOp.STAT, Order.BEFORE);
        if (status != NO_RESULT_MARKER) {
          return (FileStatus) status;
        }
      }
      FileStatus fileStatus = super.stat(path, followSymlinks);
      if (listener != null) {
        Object status = listener.accept(path, FileOp.STAT, Order.AFTER);
        if (status != NO_RESULT_MARKER) {
          return (FileStatus) status;
        }
      }
      return fileStatus;
    }

    public void scheduleMakeUnreadableAfterReaddir(final Path targetPath) {
      setCustomOverride(
          targetPath,
          new Listener() {
            @Override
            public Object accept(Path path, FileOp op, Order order) throws IOException {
              if (targetPath.equals(path) && op == FileOp.READDIR && order == Order.AFTER) {
                targetPath.setReadable(false);
              }
              return NO_RESULT_MARKER;
            }
          });
    }

    @SuppressWarnings("unchecked")
    @Override
    public Collection<Dirent> readdir(Path path, boolean followSymlinks) throws IOException {
      Listener listener = customOverrides.get(path);
      if (listener != null) {
        Object status = listener.accept(path, FileOp.READDIR, Order.BEFORE);
        if (status != NO_RESULT_MARKER) {
          return (Collection<Dirent>) status;
        }
      }
      Collection<Dirent> result = super.readdir(path, followSymlinks);
      if (listener != null) {
        Object status = listener.accept(path, FileOp.READDIR, Order.AFTER);
        if (status != NO_RESULT_MARKER) {
          return (Collection<Dirent>) status;
        }
      }
      return result;
    }
  }

  private static final Object NO_RESULT_MARKER = new Object();

  private enum Order {
    BEFORE,
    AFTER
  }

  private enum FileOp {
    STAT,
    READDIR
  }

  private interface Listener {
    Object accept(Path path, FileOp op, Order order) throws IOException;
  }
}