src/test/java/com/google/devtools/build/lib/analysis/SourceManifestActionTest.java

// Copyright 2020 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.analysis;

import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.NULL_ACTION_OWNER;

import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.actions.ActionExecutionException;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactRoot;
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
import com.google.devtools.build.lib.analysis.SourceManifestAction.ManifestType;
import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.rules.python.PythonUtils;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

/**
 * Tests for {@link SourceManifestAction}.
 */
@RunWith(JUnit4.class)
public final class SourceManifestActionTest extends BuildViewTestCase {

  private Map<PathFragment, Artifact> fakeManifest;

  private Path pythonSourcePath;
  private Artifact pythonSourceFile;
  private Path buildFilePath;
  private Artifact buildFile;

  private Path manifestOutputPath;
  private Artifact manifestOutputFile;

  @Before
  public final void createFiles() throws Exception  {
    analysisMock.pySupport().setup(mockToolsConfig);
    // Test with a raw manifest Action.
    fakeManifest = new LinkedHashMap<>();
    ArtifactRoot trivialRoot =
        ArtifactRoot.asSourceRoot(Root.fromPath(rootDirectory.getRelative("trivial")));
    buildFilePath = scratch.file("trivial/BUILD",
                                "py_binary(name='trivial', srcs =['trivial.py'])");
    buildFile = ActionsTestUtil.createArtifact(trivialRoot, buildFilePath);

    pythonSourcePath = scratch.file("trivial/trivial.py",
                                   "#!/usr/bin/python \n print 'Hello World'");
    pythonSourceFile = ActionsTestUtil.createArtifact(trivialRoot, pythonSourcePath);
    fakeManifest.put(buildFilePath.relativeTo(rootDirectory), buildFile);
    fakeManifest.put(pythonSourcePath.relativeTo(rootDirectory), pythonSourceFile);
    ArtifactRoot outputDir = ArtifactRoot.asDerivedRoot(rootDirectory, "blaze-output");
    manifestOutputPath = rootDirectory.getRelative("blaze-output/trivial.runfiles_manifest");
    manifestOutputFile = ActionsTestUtil.createArtifact(outputDir, manifestOutputPath);
  }

  /**
   * Get the contents of a file internally using an in memory output stream.
   *
   * @return returns the file contents as a string.
   * @throws ActionExecutionException
   * @throws InterruptedException
   * @throws IOException
   */
  public String getFileContentsAsString(SourceManifestAction manifest)
      throws IOException, InterruptedException, ActionExecutionException {
    ByteArrayOutputStream stream = new ByteArrayOutputStream();
    manifest.writeOutputFile(stream, reporter);
    return stream.toString();
  }

  private SourceManifestAction createSymlinkAction(Runfiles.PruningManifest pruningManifest) {
    return createAction(ManifestType.SOURCE_SYMLINKS, pruningManifest, true);
  }

  private SourceManifestAction createSymlinkAction() {
    return createSymlinkAction(null);
  }

  private SourceManifestAction createSourceOnlyAction() {
    return createAction(ManifestType.SOURCES_ONLY, null, true);
  }

  private SourceManifestAction createAction(ManifestType type,
      Runfiles.PruningManifest pruningManifest, boolean addInitPy) {
    Runfiles.Builder builder = new Runfiles.Builder("TESTING", false);
    builder.addSymlinks(fakeManifest);
    if (addInitPy) {
      builder.setEmptyFilesSupplier(PythonUtils.GET_INIT_PY_FILES);
    }
    if (pruningManifest != null) {
      builder.addPruningManifest(pruningManifest);
    }
    return new SourceManifestAction(type, NULL_ACTION_OWNER, manifestOutputFile, builder.build());
  }

  /**
   * Manifest writer that validates an expected call sequence.
   */
  private class MockManifestWriter implements SourceManifestAction.ManifestWriter {
    private List<Map.Entry<PathFragment, Artifact>> expectedSequence = new ArrayList<>();

    public MockManifestWriter() {
      expectedSequence.addAll(fakeManifest.entrySet());
    }

    @Override
    public void writeEntry(Writer manifestWriter, PathFragment rootRelativePath,
        @Nullable Artifact symlink) throws IOException {
      assertWithMessage("Expected manifest input to be exhausted").that(expectedSequence)
          .isNotEmpty();
      Map.Entry<PathFragment, Artifact> expectedEntry = expectedSequence.remove(0);
      assertThat(rootRelativePath)
          .isEqualTo(PathFragment.create("TESTING").getRelative(expectedEntry.getKey()));
      assertThat(symlink).isEqualTo(expectedEntry.getValue());
    }

    public int unconsumedInputs() {
      return expectedSequence.size();
    }

    @Override public String getMnemonic() { return null; }
    @Override public String getRawProgressMessage() { return null; }

    @Override
    public boolean isRemotable() {
      return false;
    }
  }

  /**
   * Tests that SourceManifestAction calls its manifest writer with the expected call sequence.
   */
  @Test
  public void testManifestWriterIntegration() throws Exception {
    MockManifestWriter mockWriter = new MockManifestWriter();
    getFileContentsAsString(
        new SourceManifestAction(
            mockWriter,
            NULL_ACTION_OWNER,
            manifestOutputFile,
            new Runfiles.Builder("TESTING", false).addSymlinks(fakeManifest).build()));
    assertThat(mockWriter.unconsumedInputs()).isEqualTo(0);
  }

  @Test
  public void testSimpleFileWriting() throws Exception {
    String manifestContents = getFileContentsAsString(createSymlinkAction());
    assertThat(manifestContents)
        .isEqualTo(
            "TESTING/trivial/BUILD /workspace/trivial/BUILD\n"
                + "TESTING/trivial/__init__.py \n"
                + "TESTING/trivial/trivial.py /workspace/trivial/trivial.py\n");
  }

  /**
   * Tests that the source-only formatting strategy includes relative paths only
   * (i.e. not symlinks).
   */
  @Test
  public void testSourceOnlyFormatting() throws Exception {
    String manifestContents = getFileContentsAsString(createSourceOnlyAction());
    assertThat(manifestContents)
        .isEqualTo(
            "TESTING/trivial/BUILD\n"
                + "TESTING/trivial/__init__.py\n"
                + "TESTING/trivial/trivial.py\n");
  }

  /**
   * Test that a directory which has only a .so file in the manifest triggers
   * the inclusion of a __init__.py file for that directory.
   */
  @Test
  public void testSwigLibrariesTriggerInitDotPyInclusion() throws Exception {
    ArtifactRoot swiggedLibPath =
        ArtifactRoot.asSourceRoot(Root.fromPath(rootDirectory.getRelative("swig")));
    Path swiggedFile = scratch.file("swig/fakeLib.so");
    Artifact swigDotSO = ActionsTestUtil.createArtifact(swiggedLibPath, swiggedFile);
    fakeManifest.put(swiggedFile.relativeTo(rootDirectory), swigDotSO);
    String manifestContents = getFileContentsAsString(createSymlinkAction());
    assertThat(manifestContents).containsMatch(".*TESTING/swig/__init__.py .*");
    assertThat(manifestContents).containsMatch("fakeLib.so");
  }

  @Test
  public void testNoPythonOrSwigLibrariesDoNotTriggerInitDotPyInclusion() throws Exception {
    ArtifactRoot nonPythonPath =
        ArtifactRoot.asSourceRoot(Root.fromPath(rootDirectory.getRelative("not_python")));
    Path nonPythonFile = scratch.file("not_python/blob_of_data");
    Artifact nonPython = ActionsTestUtil.createArtifact(nonPythonPath, nonPythonFile);
    fakeManifest.put(nonPythonFile.relativeTo(rootDirectory), nonPython);
    String manifestContents = getFileContentsAsString(createSymlinkAction());
    assertThat(manifestContents).doesNotContain("not_python/__init__.py \n");
    assertThat(manifestContents).containsMatch("blob_of_data");
  }

  @Test
  public void testGetMnemonic() throws Exception {
    assertThat(createSymlinkAction().getMnemonic()).isEqualTo("SourceSymlinkManifest");
    assertThat(createAction(ManifestType.SOURCE_SYMLINKS, null, false).getMnemonic())
        .isEqualTo("SourceSymlinkManifest");
    assertThat(createSourceOnlyAction().getMnemonic()).isEqualTo("PackagingSourcesManifest");
  }

  @Test
  public void testSymlinkProgressMessage() throws Exception {
    String progress = createSymlinkAction().getProgressMessage();
    assertWithMessage("null action not found in " + progress)
        .that(progress.contains("//null/action:owner"))
        .isTrue();
  }

  @Test
  public void testSymlinkProgressMessageNoPyInitFiles() throws Exception {
    String progress =  createAction(ManifestType.SOURCE_SYMLINKS, null, false).getProgressMessage();
    assertWithMessage("null action not found in " + progress)
        .that(progress.contains("//null/action:owner"))
        .isTrue();
  }

  @Test
  public void testSourceOnlyProgressMessage() throws Exception {
    SourceManifestAction action =
        new SourceManifestAction(
            ManifestType.SOURCES_ONLY,
            NULL_ACTION_OWNER,
            getBinArtifactWithNoOwner("trivial.runfiles_manifest"),
            Runfiles.EMPTY);
    String progress = action.getProgressMessage();
    assertWithMessage("null action not found in " + progress)
        .that(progress.contains("//null/action:owner"))
        .isTrue();
  }

  @Test
  public void testRootSymlinksAffectKey() throws Exception {
    Artifact manifest1 = getBinArtifactWithNoOwner("manifest1");
    Artifact manifest2 = getBinArtifactWithNoOwner("manifest2");

    SourceManifestAction action1 =
        new SourceManifestAction(
            ManifestType.SOURCE_SYMLINKS,
            NULL_ACTION_OWNER,
            manifest1,
            new Runfiles.Builder("TESTING", false)
                .addRootSymlinks(ImmutableMap.of(PathFragment.create("a"), buildFile))
                .build());

    SourceManifestAction action2 =
        new SourceManifestAction(
            ManifestType.SOURCE_SYMLINKS,
            NULL_ACTION_OWNER,
            manifest2,
            new Runfiles.Builder("TESTING", false)
                .addRootSymlinks(ImmutableMap.of(PathFragment.create("b"), buildFile))
                .build());

    assertThat(computeKey(action2)).isNotEqualTo(computeKey(action1));
  }

  // Regression test for b/116254698.
  @Test
  public void testEmptyFilesAffectKey() throws Exception {
    Artifact manifest1 = getBinArtifactWithNoOwner("manifest1");
    Artifact manifest2 = getBinArtifactWithNoOwner("manifest2");

    SourceManifestAction action1 =
        new SourceManifestAction(
            ManifestType.SOURCE_SYMLINKS,
            NULL_ACTION_OWNER,
            manifest1,
            new Runfiles.Builder("TESTING", false)
                .addSymlink(PathFragment.create("a"), buildFile)
                .setEmptyFilesSupplier(
                    paths ->
                        paths.stream()
                            .map(p -> p.replaceName(p.getBaseName() + "~"))
                            .collect(Collectors.toSet()))
                .build());

    SourceManifestAction action2 =
        new SourceManifestAction(
            ManifestType.SOURCE_SYMLINKS,
            NULL_ACTION_OWNER,
            manifest2,
            new Runfiles.Builder("TESTING", false)
                .addSymlink(PathFragment.create("a"), buildFile)
                .setEmptyFilesSupplier(
                    paths ->
                        paths.stream()
                            .map(p -> p.replaceName(p.getBaseName() + "~~"))
                            .collect(Collectors.toSet()))
                .build());

    assertThat(computeKey(action2)).isNotEqualTo(computeKey(action1));
  }

  /**
   * Returns a new pruning manifest with the given manifest file and set of candidate source
   * artifacts.
   */
  private Runfiles.PruningManifest pruningManifest(Artifact manifestFile, String... candidates)
      throws Exception {
    NestedSetBuilder<Artifact> builder = NestedSetBuilder.stableOrder();
    for (String name : candidates) {
      builder.add(getSourceArtifact(name));
    }
    return new Runfiles.PruningManifest(builder.build(), manifestFile);
  }

  /**
   * Constructs a new manifest file artifact with the given name, writes the given contents
   * to that file, and returns the artifact.
   */
  private Artifact manifestFile(String name, String... lines) throws Exception {
    Artifact artifact = getBinArtifactWithNoOwner(name);
    scratch.file(artifact.getPath().getPathString(), lines);
    return artifact;
  }

  /**
   * Tests that pruning manifest candidates are conditionally included, depending on whether
   * or not they appear in the manifest.
   */
  @Test
  public void testPruningManifest() throws Exception {
    Artifact manifestFile = manifestFile("pruned.manifest", "a/b2.txt", "a/b4.txt");
    Runfiles.PruningManifest manifest =
        pruningManifest(manifestFile, "a/b1.txt", "a/b2.txt", "a/b3.txt", "a/b4.txt");
    String manifestContents = getFileContentsAsString(createSymlinkAction(manifest));

    assertThat(manifestContents)
        .isEqualTo(
            "TESTING/a/b2.txt /workspace/a/b2.txt\n"
                + "TESTING/a/b4.txt /workspace/a/b4.txt\n"
                + "TESTING/trivial/BUILD /workspace/trivial/BUILD\n"
                + "TESTING/trivial/__init__.py \n"
                + "TESTING/trivial/trivial.py /workspace/trivial/trivial.py\n");
  }

  /**
   * Tests that the pruning manifest can't add runfiles that aren't part of the candidate set.
   */
  @Test
  public void testPruningManifestOnlyExcludes() throws Exception {
    Artifact manifestFile = manifestFile("pruned.manifest", "a/b2.txt", "a/b_UNDECLARED.txt");
    Runfiles.PruningManifest manifest =
        pruningManifest(manifestFile, "a/b1.txt", "a/b2.txt", "a/b3.txt", "a/b4.txt");
    String manifestContents = getFileContentsAsString(createSymlinkAction(manifest));

    assertThat(manifestContents)
        .isEqualTo(
            "TESTING/a/b2.txt /workspace/a/b2.txt\n"
                + "TESTING/trivial/BUILD /workspace/trivial/BUILD\n"
                + "TESTING/trivial/__init__.py \n"
                + "TESTING/trivial/trivial.py /workspace/trivial/trivial.py\n");
  }

  /**
   * Tests that the pruning manifest can't exclude runfiles that are also included from other
   * sources.
   */
  @Test
  public void testPruningManifestDoesntOverrideExplicitArtifacts() throws Exception {
    Artifact manifestFile = manifestFile("pruned.manifest", "a/include.txt");
    Runfiles.PruningManifest manifest =
        pruningManifest(manifestFile, "a/include.txt", "a/exclude.txt", "trivial/trivial.py");
    String manifestContents = getFileContentsAsString(createSymlinkAction(manifest));

    assertThat(manifestContents)
        .isEqualTo(
            "TESTING/a/include.txt /workspace/a/include.txt\n"
                + "TESTING/trivial/BUILD /workspace/trivial/BUILD\n"
                + "TESTING/trivial/__init__.py \n"
                + "TESTING/trivial/trivial.py /workspace/trivial/trivial.py\n");
  }

  private String computeKey(SourceManifestAction action) {
    Fingerprint fp = new Fingerprint();
    action.computeKey(actionKeyContext, fp);
    return fp.hexDigestAndReset();
  }
}