src/test/java/com/google/devtools/build/lib/skyframe/ActionOutputDirectoryHelperTest.java

// Copyright 2021 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;

import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.truth.Truth.assertThat;
import static com.google.devtools.build.lib.testing.common.DirectoryListingHelper.leafDirectoryEntries;
import static org.junit.Assert.assertThrows;

import com.github.benmanes.caffeine.cache.CaffeineSpec;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.actions.ArtifactRoot;
import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
import com.google.devtools.build.lib.skyframe.ActionOutputDirectoryHelper.CreateOutputDirectoryException;
import com.google.devtools.build.lib.testing.common.DirectoryListingHelper;
import com.google.devtools.build.lib.testutil.Scratch;
import com.google.devtools.build.lib.vfs.DelegateFileSystem;
import com.google.devtools.build.lib.vfs.Dirent;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.testing.junit.testparameterinjector.TestParameter;
import com.google.testing.junit.testparameterinjector.TestParameterInjector;
import java.io.IOException;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;

/** Tests for {@link ActionOutputDirectoryHelper}. */
@RunWith(TestParameterInjector.class)
public class ActionOutputDirectoryHelperTest {

  private Path execRoot;
  private ArtifactRoot outputRoot;

  @Before
  public void createArtifactRootAndOutputDirectoryHelper() throws IOException {
    Scratch scratch = new Scratch();
    execRoot = scratch.dir("/execroot");
    outputRoot = createOutputRoot(execRoot);
  }

  enum DirectoryCache {
    CACHE_ENABLED(CaffeineSpec.parse("maximumSize=100000")),
    CACHE_DISABLED(CaffeineSpec.parse("maximumSize=0"));

    @SuppressWarnings("ImmutableEnumChecker")
    final CaffeineSpec spec;

    DirectoryCache(CaffeineSpec spec) {
      this.spec = spec;
    }
  }

  private enum OutputSet {
    SINGLE_FILE(
        /*fileOutputs=*/ ImmutableSet.of("a/b"),
        /*treeOutputs=*/ ImmutableSet.of(),
        /*expectedDirectories=*/ ImmutableList.of("a")),
    DEEP_DIRECTORY_STRUCTURE(
        /*fileOutputs=*/ ImmutableSet.of("a/b/c/d/e/f/g/h/i/j/k/l/m"),
        /*treeOutputs=*/ ImmutableSet.of(),
        /*expectedDirectories=*/ ImmutableList.of("a/b/c/d/e/f/g/h/i/j/k/l")),
    MULTIPLE_FILES(
        /*fileOutputs=*/ ImmutableSet.of("a/b/c", "a/c", "a/d/1", "a/d/2"),
        /*treeOutputs=*/ ImmutableSet.of(),
        /*expectedDirectories=*/ ImmutableList.of("a/b", "a/d")),
    TREE_OUTPUT(
        /*fileOutputs=*/ ImmutableSet.of(),
        /*treeOutputs=*/ ImmutableSet.of("a/b"),
        /*expectedDirectories=*/ ImmutableList.of("a/b"));

    OutputSet(
        ImmutableSet<String> fileOutputs,
        ImmutableSet<String> treeOutputs,
        ImmutableList<String> expectedDirectories) {
      this.fileOutputs = fileOutputs;
      this.treeOutputs = treeOutputs;
      this.expectedDirectories = expectedDirectories;
    }

    ImmutableSet<Artifact> actionOutputs(ActionOutputDirectoryHelperTest test) {
      ImmutableSet.Builder<Artifact> outs =
          ImmutableSet.builderWithExpectedSize(fileOutputs.size() + treeOutputs.size());
      fileOutputs.stream().map(test::createOutput).forEach(outs::add);
      treeOutputs.stream().map(test::createTreeOutput).forEach(outs::add);
      return outs.build();
    }

    ImmutableList<Dirent> expectedDirectoryEntries() {
      return expectedDirectories.stream()
          .map(DirectoryListingHelper::directory)
          .collect(toImmutableList());
    }

    private final ImmutableSet<String> fileOutputs;
    private final ImmutableSet<String> treeOutputs;
    private final ImmutableList<String> expectedDirectories;
  }

  @Test
  public void createOutputDirectories_createsExpectedDirectories(
      @TestParameter DirectoryCache cache, @TestParameter OutputSet outputSet) throws Exception {
    ActionOutputDirectoryHelper outputDirectoryHelper = new ActionOutputDirectoryHelper(cache.spec);

    outputDirectoryHelper.createOutputDirectories(outputSet.actionOutputs(this));

    assertThat(leafDirectoryEntries(outputRoot.getRoot().asPath()))
        .containsExactlyElementsIn(outputSet.expectedDirectoryEntries());
  }

  @Test
  public void createOutputDirectories_makesOutputDirectoryWritable() throws Exception {
    ActionOutputDirectoryHelper outputDirectoryHelper = createActionOutputDirectoryHelper();
    Artifact fileOutput = createOutput("dir/file");
    Path parentDir = fileOutput.getPath().getParentDirectory();
    parentDir.createDirectoryAndParents();
    parentDir.setWritable(false);
    parentDir.setExecutable(false);

    outputDirectoryHelper.createOutputDirectories(ImmutableSet.of(fileOutput));

    assertThat(parentDir.isReadable()).isTrue();
    assertThat(parentDir.isWritable()).isTrue();
    assertThat(parentDir.isExecutable()).isTrue();
  }

  @Test
  public void createActionFsOutputDirectories_createsExpectedDirectoriesInActionFs(
      @TestParameter OutputSet outputSet) throws Exception {
    ActionOutputDirectoryHelper outputDirectoryHelper = createActionOutputDirectoryHelper();
    FileSystem actionFileSystem = new Scratch().getFileSystem();
    ArtifactPathResolver resolver =
        ArtifactPathResolver.createPathResolver(actionFileSystem, execRoot);

    outputDirectoryHelper.createActionFsOutputDirectories(outputSet.actionOutputs(this), resolver);

    Path outputRootPath = outputRoot.getRoot().asPath();
    assertThat(outputRootPath.exists()).isFalse();
    assertThat(leafDirectoryEntries(actionFileSystem.getPath(outputRootPath.asFragment())))
        .containsExactlyElementsIn(outputSet.expectedDirectoryEntries());
  }

  @Test
  public void createOutputDirectories_ioExceptionWhenCreatingDirectory_fails(
      @TestParameter DirectoryCache cache) {
    ActionOutputDirectoryHelper outputDirectoryHelper = new ActionOutputDirectoryHelper(cache.spec);
    IOException injectedException = new IOException("oh no!");
    PathFragment outputRootPath = outputRoot.getRoot().asPath().asFragment();
    FileSystem fsWithFailures =
        createFileSystemInjectingException(outputRootPath.getRelative("dir"), injectedException);
    ArtifactRoot rootWithFailure = createOutputRoot(fsWithFailures.getPath(execRoot.asFragment()));
    ImmutableSet<Artifact> outputs = ImmutableSet.of(createOutput(rootWithFailure, "dir/file"));

    CreateOutputDirectoryException e =
        assertThrows(
            CreateOutputDirectoryException.class,
            () -> outputDirectoryHelper.createOutputDirectories(outputs));

    assertThat(e.directoryPath).isEqualTo(outputRootPath.getRelative("dir"));
    assertThat(e).hasCauseThat().isSameInstanceAs(injectedException);
  }

  @Test
  public void createActionFsOutputDirectories_ioExceptionWhenCreatingDirectory_fails() {
    ActionOutputDirectoryHelper outputDirectoryHelper = createActionOutputDirectoryHelper();
    IOException injectedException = new IOException("oh no!");
    PathFragment outputRootPath = outputRoot.getRoot().asPath().asFragment();
    FileSystem fsWithFailures =
        createFileSystemInjectingException(outputRootPath.getRelative("dir"), injectedException);
    ArtifactRoot rootWithFailure = createOutputRoot(fsWithFailures.getPath(execRoot.asFragment()));
    ImmutableSet<Artifact> outputs = ImmutableSet.of(createOutput(rootWithFailure, "dir/file"));
    FileSystem actionFileSystem = new DelegateFileSystem(fsWithFailures) {};
    ArtifactPathResolver pathResolver =
        ArtifactPathResolver.createPathResolver(actionFileSystem, execRoot);

    CreateOutputDirectoryException e =
        assertThrows(
            CreateOutputDirectoryException.class,
            () -> outputDirectoryHelper.createActionFsOutputDirectories(outputs, pathResolver));

    assertThat(e.directoryPath).isEqualTo(outputRootPath.getRelative("dir"));
    assertThat(e).hasCauseThat().isSameInstanceAs(injectedException);
  }

  @Test
  public void invalidateTreeArtifactDirectoryCreation_onlyInvalidatesTreeArtifactDirs()
      throws Exception {
    ActionOutputDirectoryHelper outputDirectoryHelper = createActionOutputDirectoryHelper();
    Artifact regularOutput = createOutput("example/regular/file.txt");
    SpecialArtifact treeOutput = createTreeOutput("example/tree/tree_dir");
    ImmutableSet<Artifact> outputs = ImmutableSet.of(regularOutput, treeOutput);

    outputDirectoryHelper.createOutputDirectories(outputs);
    regularOutput.getPath().getParentDirectory().deleteTree();
    treeOutput.getPath().deleteTree();
    outputDirectoryHelper.invalidateTreeArtifactDirectoryCreation(outputs);
    outputDirectoryHelper.createOutputDirectories(outputs);

    // Only tree artifact directories are recreated.
    assertThat(regularOutput.getPath().getParentDirectory().exists()).isFalse();
    assertThat(treeOutput.getPath().exists()).isTrue();
  }

  private FileSystem createFileSystemInjectingException(
      PathFragment failingPath, IOException injectedException) {
    return new DelegateFileSystem(execRoot.getFileSystem()) {
      @Override
      public boolean createDirectory(PathFragment path) throws IOException {
        if (path.equals(failingPath)) {
          throw injectedException;
        }
        return super.createDirectory(path);
      }

      @Override
      public void createDirectoryAndParents(PathFragment path) throws IOException {
        if (path.equals(failingPath)) {
          throw injectedException;
        }
        super.createDirectoryAndParents(path);
      }
    };
  }

  private SpecialArtifact createTreeOutput(String relativeExecPath) {
    return ActionsTestUtil.createTreeArtifactWithGeneratingAction(
        outputRoot, outputRoot.getExecPath().getRelative(relativeExecPath));
  }

  private Artifact createOutput(String relativeExecPath) {
    return createOutput(outputRoot, relativeExecPath);
  }

  private static Artifact createOutput(ArtifactRoot outputRoot, String relativeExecPath) {
    return ActionsTestUtil.createArtifactWithRootRelativePath(
        outputRoot, PathFragment.create(relativeExecPath));
  }

  private static ArtifactRoot createOutputRoot(Path execRoot) {
    return ArtifactRoot.asDerivedRoot(execRoot, RootType.Output, "out");
  }

  private static ActionOutputDirectoryHelper createActionOutputDirectoryHelper() {
    return new ActionOutputDirectoryHelper(DirectoryCache.CACHE_ENABLED.spec);
  }
}