src/test/java/com/google/devtools/build/lib/buildtool/BuildResultTestCase.java

// Copyright 2020 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.buildtool;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.fail;

import com.google.devtools.build.lib.actions.BuildFailedException;
import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
import com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase;
import com.google.devtools.build.lib.skyframe.BuildResultListener;
import com.google.devtools.build.lib.util.io.OutErr;
import com.google.devtools.build.lib.util.io.RecordingOutErr;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

/** Tests correctness of the build summary output produced by BuildTool.showBuildResult() method. */
public abstract class BuildResultTestCase extends BuildIntegrationTestCase {
  private RecordingOutErr recOutErr = new RecordingOutErr();

  private static final String GENRULE_ERROR = "Exit 42";

  /**
   * Hook for subclasses to define which executor we use.  (The two concrete
   * subclasses, {Sequential,Parallel}BuildResultTest, are at the bottom of this
   * source file.)
   */
  protected abstract int numJobs();

  @Before
  public final void addOptions() throws Exception  {
    this.outErr = recOutErr;
    addOptions("--jobs", "" + numJobs(), "--show_result", "1000");
  }

  private void build(boolean expectFailure, String expectedError, String... targets)
      throws Exception {
    try {
      buildTarget(targets);
      if (expectFailure) {
        fail();
      }
    } catch (BuildFailedException e) {
      if (e.getMessage() != null) {
        assertThat(e).hasMessageThat().containsMatch(expectedError);
      }
    } finally {
      OutErr.SYSTEM_OUT_ERR.printErr(recOutErr.errAsLatin1());
      OutErr.SYSTEM_OUT_ERR.printOut(recOutErr.outAsLatin1());
    }
  }

  @Test
  public void testKeepGoingResult() throws Exception {
    write("test/BUILD",
        "genrule(name='A', srcs=['A1','A2'], outs=['A.out']," +
        " cmd='/bin/cp test/in $(location A.out)')\n" +
        "genrule(name='A1', srcs=[], outs=['A1.out']," +
        " cmd='/bin/cp test/in $(location A1.out)')\n" +
        "genrule(name='A2', srcs=[], outs=['A2.out']," +
        " cmd='exit 42')\n" +
        "genrule(name='B', srcs=[], outs=['B.out']," +
        " cmd='/bin/cp test/in $(location B.out)')\n");
    write("test/in", "(input)");

    addOptions("--keep_going");
    build(true, GENRULE_ERROR, "//test:B", "//test:A");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).contains("Target //test:A failed to build\n");
    assertThat(stderr).containsMatch("Target //test:B up-to-date:\n" + "  .*/test/B\\.out\n");
  }

  @Test
  public void testWithOnlyInternalOutputGroups() throws Exception {
    write(
        "foo/BUILD",
        """
        load("//test_defs:foo_binary.bzl", "foo_binary")

        genrule(
            name = "gen",
            outs = ["gen.out"],
            cmd = "touch $@",
        )

        foo_binary(
            name = "foo",
            srcs = ["foo.sh"],
            data = [":gen"],
        )
        """);
    write("foo/foo.sh", "echo foo").setExecutable(true);

    // Subtracting the default output group leaves the internal output group carrying the runfiles,
    // which are built but never shown.
    addOptions("--output_groups=-default");
    build(false, "no-error", "//foo");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr)
        .contains("Target //foo:foo up-to-date (nothing to build except internal output groups)\n");
  }

  /**
   * Writes {@code //groups:t}, whose default output is {@code t_default.txt} and whose {@code
   * custom} output group contains {@code t_custom.txt} and {@code t_shared.txt}. {@code
   * t_shared.txt} is also in the {@code shared} output group.
   */
  private void writeOutputGroupsTarget() throws Exception {
    write(
        "groups/defs.bzl",
        """
        def _impl(ctx):
            default = ctx.actions.declare_file(ctx.label.name + "_default.txt")
            custom = ctx.actions.declare_file(ctx.label.name + "_custom.txt")
            shared = ctx.actions.declare_file(ctx.label.name + "_shared.txt")
            for f in [default, custom, shared]:
                ctx.actions.write(f, f.basename)
            return [
                DefaultInfo(files = depset([default])),
                OutputGroupInfo(
                    custom = depset([custom, shared]),
                    shared = depset([shared]),
                ),
            ]

        groups_rule = rule(implementation = _impl)
        """);
    write(
        "groups/BUILD",
        """
        load(":defs.bzl", "groups_rule")

        groups_rule(name = "t")
        """);
  }

  @Test
  public void testHideOutputGroupResults() throws Exception {
    writeOutputGroupsTarget();

    addOptions("--output_groups=+custom", "--hide_output_group_results=custom");
    build(false, "no-error", "//groups:t");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr)
        .containsMatch("Target //groups:t up-to-date:\n  .*/groups/t_default\\.txt\n");
    assertThat(stderr).doesNotContain("t_custom.txt");
    assertThat(stderr).doesNotContain("t_shared.txt");
  }

  @Test
  public void testHideOutputGroupResults_defaultOutputGroup() throws Exception {
    writeOutputGroupsTarget();

    addOptions("--output_groups=+custom", "--hide_output_group_results=default");
    build(false, "no-error", "//groups:t");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).doesNotContain("t_default.txt");
    assertThat(stderr).contains("/groups/t_custom.txt\n");
    assertThat(stderr).contains("/groups/t_shared.txt\n");
  }

  @Test
  public void testHideOutputGroupResults_artifactInVisibleGroupStillShown() throws Exception {
    writeOutputGroupsTarget();

    addOptions("--output_groups=+custom,+shared", "--hide_output_group_results=custom");
    build(false, "no-error", "//groups:t");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).contains("/groups/t_default.txt\n");
    assertThat(stderr).contains("/groups/t_shared.txt\n");
    assertThat(stderr).doesNotContain("t_custom.txt");
  }

  @Test
  public void testHideOutputGroupResults_suppressesUnknownOutputGroupWarning() throws Exception {
    writeOutputGroupsTarget();

    addOptions(
        "--output_groups=+hidden_missing,+shown_missing",
        "--hide_output_group_results=hidden_missing");
    build(false, "no-error", "//groups:t");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).doesNotContain("Output group 'hidden_missing' was requested");
    assertThat(stderr).contains("Output group 'shown_missing' was requested");
  }

  @Test
  public void testHideOutputGroupResults_doesNotSuppressUnknownOutputGroupError() throws Exception {
    writeOutputGroupsTarget();

    addOptions(
        "--output_groups=+hidden_missing",
        "--hide_output_group_results=hidden_missing",
        "--incompatible_fail_on_unknown_output_groups");
    ViewCreationFailedException e =
        assertThrows(ViewCreationFailedException.class, () -> buildTarget("//groups:t"));
    assertThat(e).hasMessageThat().contains("Output group 'hidden_missing' was requested");
  }

  @Test
  public void testNoKeepGoingResult() throws Exception {
    write("test/BUILD",
        "genrule(name='A', srcs=['A2'], outs=['A.out']," +
        " cmd='/bin/cp test/in $(location A.out)')\n" +
        "genrule(name='A2', srcs=[], outs=['A2.out']," +
        " cmd='exit 42')\n" +
        "genrule(name='B', srcs=[], outs=['B.out']," +
        " cmd='/bin/cp test/in $(location B.out)')\n");
    write("test/in", "(input)");

    build(true, GENRULE_ERROR, "//test:A", "//test:B");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).containsMatch("Target //test:A failed to build\n");
  }

  /**
   * This is a regression test for bug #823077, in which presence of artifacts
   * with null generating action caused targets always to be reported as failed.
   */
  @Test
  public void testKeepGoingResultWithNullActions() throws Exception {
    write("foo/BUILD",
          "sh_test(name='test_bar', srcs=['test_bar.sh'])\n" +
          "sh_test(name='test_foo', srcs=['test_foo.sh'])\n");
    write("foo/test_bar.sh", "echo bar is fine").setExecutable(true);

    addOptions("--keep_going");
    build(true, "no-error", "//foo:test_foo", "//foo:test_bar");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).contains("Target //foo:test_bar up-to-date:\n");
    assertThat(stderr).containsMatch("\n  .*/foo/test_bar\n");
    assertThat(stderr).doesNotContainMatch("\n  .*/foo/test_bar\\.sh\n");
    assertThat(stderr).contains("Target //foo:test_foo failed to build\n");
  }

  /**
   * This is a regression test for bug #1044470, in which targets with failed data dependencies
   * were still being reported as "up-to-date".
   */
  @Test
  public void testWithMissingData() throws Exception {
    write(
        "needsdata/BUILD",
        """
        load("@rules_cc//cc:cc_library.bzl", "cc_library")
        cc_library(
            name = "needsdata",
            data = [":data_lib"],
        )

        cc_library(
            name = "data_lib",
            data = [":does_not_exist"],
        )
        """);

    // TODO(bazel-team): figure out why error message is non-deterministic with Skyframe full.
    // LOADING_AND_ANALYSIS loading_and_analysis cleanup.
    build(true, "input", "//needsdata");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).doesNotContain("Target //needsdata:needsdata up-to-date:\n");
    assertThat(stderr).contains("Target //needsdata:needsdata failed to build\n");
    assertThat(stderr)
        .doesNotContain("Use --verbose_failures to see the command lines of failed build steps.\n");
  }

  /**
   * This is a regression test for bug #987608, in which the temp artifacts (.ii and .s)
   * were not being reported as having been created.
   */
  @Test
  public void testWithSaveTemps() throws Exception {
    write(
        "my_clib/BUILD",
        "load('@rules_cc//cc:cc_library.bzl', 'cc_library')",
        "cc_library(name='my_clib', srcs=['myclib.cc'])\n");
    write("my_clib/myclib.cc",
          "void f() {}");

    addOptions("--save_temps");
    build(false, "no-error", "//my_clib");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).contains("Target //my_clib:my_clib up-to-date:\n");
    assertThat(stderr).contains("blaze-bin/my_clib/_objs/my_clib/myclib.pic.s\n");
    assertThat(stderr).contains("blaze-bin/my_clib/_objs/my_clib/myclib.pic.ii\n");
  }

  /**
   * Test that changing the symlink prefix builds into another directory.
   */
  @Test
  public void testSymlinkPrefix() throws Exception {
    write(
        "my_clib/BUILD",
        "load('@rules_cc//cc:cc_library.bzl', 'cc_library')",
        "cc_library(name='my_clib', srcs=['myclib.cc'])\n");
    write("my_clib/myclib.cc",
          "void f() {}");

    addOptions("--symlink_prefix=myblaze-");
    build(false, "no-error", "//my_clib");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).contains("Target //my_clib:my_clib up-to-date:\n");
    assertThat(stderr).contains("myblaze-bin/my_clib/libmy_clib.so\n");
    assertThat(stderr).contains("myblaze-bin/my_clib/libmy_clib.a\n");
  }

  /**
   * This is a regression test for bug #1013874, in which the temp artifacts (.ii and .s)
   * were not being reported as having been created within a failing target.
   * It's possible that one or more temps were successfully created, even if the
   * compilation failed.
   */
  @Test
  public void testFailedTargetWithSaveTemps() throws Exception {
    write(
        "bad_clib/BUILD",
        "load('@rules_cc//cc:cc_library.bzl', 'cc_library')",
        "cc_library(name='bad_clib', srcs=['badlib.cc'])\n");
    // trigger a warning to make the build fail:
    //   "control reaches end of non-void function [-Werror,-Wreturn-type]"
    write("bad_clib/badlib.cc",
        "int f() { }");

    // We need to set --keep_going so that the temps get built even though the compilation fails.
    addOptions("--save_temps", "--keep_going");
    build(true, "compilation of rule '//bad_clib:bad_clib' failed", "//bad_clib");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).contains("Target //bad_clib:bad_clib failed to build");
    assertThat(stderr).contains("See temp at blaze-bin/bad_clib/_objs/bad_clib/badlib.pic.ii\n");
  }

  @Test
  public void testFailedDependencyActionAttributionAndShowResult() throws Exception {
    write(
        "test/BUILD",
        "genrule(name='top1', srcs=['dep'], outs=['top1.out'],"
            + " cmd='/bin/cp test/in $(location top1.out)')\n"
            + "genrule(name='top2', srcs=['dep'], outs=['top2.out'],"
            + " cmd='/bin/cp test/in $(location top2.out)')\n"
            + "genrule(name='dep', srcs=[], outs=['dep.out'],"
            + " cmd='exit 42')\n");
    write("test/in", "(input)");

    addOptions("--keep_going", "--show_result=2");
    build(true, GENRULE_ERROR, "//test:top1", "//test:top2");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).contains("Target //test:top1 failed to build\n");
    assertThat(stderr).contains("Target //test:top2 failed to build\n");
    assertThat(stderr).contains("  due to action in //test:dep\n");
  }

  @Test
  public void testFailedTargetsSuppressedWhenExceedingShowResult() throws Exception {
    write(
        "test/BUILD",
        "genrule(name='top1', srcs=['dep'], outs=['top1.out'],"
            + " cmd='/bin/cp test/in $(location top1.out)')\n"
            + "genrule(name='top2', srcs=['dep'], outs=['top2.out'],"
            + " cmd='/bin/cp test/in $(location top2.out)')\n"
            + "genrule(name='dep', srcs=[], outs=['dep.out'],"
            + " cmd='exit 42')\n");
    write("test/in", "(input)");

    addOptions("--keep_going", "--show_result=1");
    build(true, GENRULE_ERROR, "//test:top1", "//test:top2");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).doesNotContain("Target //test:top1 failed to build\n");
    assertThat(stderr).doesNotContain("Target //test:top2 failed to build\n");
    assertThat(stderr)
        .contains("Use --verbose_failures to see the command lines of failed build steps.\n");
  }

  @Test
  public void testNoKeepGoingUnbuiltTargetsExcludedFromFailedBudget() throws Exception {
    write(
        "test/defs.bzl",
        "def _fail_rule_impl(ctx):\n"
            + "  out = ctx.actions.declare_file(ctx.label.name + '.out')\n"
            + "  ctx.actions.run_shell(outputs = [out], inputs = [], tools = [], command = 'exit"
            + " 42')\n"
            + "  return [DefaultInfo(files = depset([out]))]\n"
            + "fail_rule = rule(implementation = _fail_rule_impl)\n"
            + "def _pass_rule_impl(ctx):\n"
            + "  out = ctx.actions.declare_file(ctx.label.name + '.out')\n"
            + "  ctx.actions.run_shell(outputs = [out], inputs = ctx.files.srcs, tools = [],"
            + " command = 'touch ' + out.path)\n"
            + "  return [DefaultInfo(files = depset([out]))]\n"
            + "pass_rule = rule(implementation = _pass_rule_impl, attrs = {'srcs':"
            + " attr.label_list(allow_files = True)})\n");
    write(
        "test/BUILD",
        "load(':defs.bzl', 'fail_rule', 'pass_rule')\n"
            + "pass_rule(name='top1', srcs=[':dep'])\n"
            + "fail_rule(name='dep')\n"
            + "genrule(name='unbuilt', srcs=['u1'], outs=['unbuilt.out'],"
            + " cmd='/bin/cp $(location u1) $(location unbuilt.out)')\n"
            + "genrule(name='u1', srcs=['u2'], outs=['u1.out'],"
            + " cmd='/bin/cp $(location u2) $(location u1.out)')\n"
            + "genrule(name='u2', srcs=['u3'], outs=['u2.out'],"
            + " cmd='/bin/cp $(location u3) $(location u2.out)')\n"
            + "genrule(name='u3', srcs=['in'], outs=['u3.out'],"
            + " cmd='/bin/cp $(location in) $(location u3.out)')\n");
    write("test/in", "(input)");

    addOptions("--nokeep_going", "--show_result=1");
    build(true, GENRULE_ERROR, "//test:top1", "//test:unbuilt");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).contains("Target //test:top1 failed to build\n");
    assertThat(stderr).contains("  due to action in //test:dep\n");
    assertThat(stderr).doesNotContain("Target //test:unbuilt failed to build\n");
    assertThat(stderr).doesNotContain("Target //test:unbuilt up-to-date");
  }

  @Test
  public void testShowResultZeroSuppressesFailedTargets() throws Exception {
    write("test/BUILD", "genrule(name='fail', srcs=[], outs=['fail.out'], cmd='exit 42')\n");

    addOptions("--show_result=0");
    build(true, GENRULE_ERROR, "//test:fail");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).doesNotContain("Target //test:fail failed to build\n");
    assertThat(stderr)
        .contains("Use --verbose_failures to see the command lines of failed build steps.\n");
  }

  @Test
  public void testFailedTargetNotSuppressedBySuccessfulTargetBudget() throws Exception {
    write(
        "mix/BUILD",
        "genrule(name='succ1', srcs=['in'], outs=['succ1.out'], cmd='/bin/cp $(location in)"
            + " $(location succ1.out)')\n"
            + "genrule(name='succ2', srcs=['in'], outs=['succ2.out'], cmd='/bin/cp $(location in)"
            + " $(location succ2.out)')\n"
            + "genrule(name='fail', srcs=[], outs=['fail.out'], cmd='exit 42')\n");
    write("mix/in", "(input)");

    addOptions("--keep_going", "--show_result=1");
    build(true, GENRULE_ERROR, "//mix:succ1", "//mix:succ2", "//mix:fail");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).contains("Target //mix:fail failed to build\n");
    assertThat(stderr).doesNotContain("Target //mix:succ1 up-to-date:\n");
    assertThat(stderr).doesNotContain("Target //mix:succ2 up-to-date:\n");
  }

  @Test
  public void testActionFailureShowsVerboseFailuresSuggestion() throws Exception {
    write("test/BUILD", "genrule(name='fail', srcs=[], outs=['fail.out'], cmd='exit 42')\n");

    build(true, GENRULE_ERROR, "//test:fail");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).contains("Target //test:fail failed to build\n");
    assertThat(stderr)
        .contains("Use --verbose_failures to see the command lines of failed build steps.\n");
    assertThat(stderr).doesNotContain("Use --sandbox_debug");
  }

  @Test
  public void testSandboxDebugSuppressesVerboseFailuresSuggestion() throws Exception {
    write("test/BUILD", "genrule(name='fail', srcs=[], outs=['fail.out'], cmd='exit 42')\n");

    addOptions("--sandbox_debug");
    build(true, GENRULE_ERROR, "//test:fail");

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr).contains("Target //test:fail failed to build\n");
    assertThat(stderr)
        .doesNotContain("Use --verbose_failures to see the command lines of failed build steps.\n");
  }

  @Test
  public void testSandboxedActionFailureShowsSandboxDebugSuggestion() throws Exception {
    write("test/BUILD", "genrule(name='fail', srcs=[], outs=['fail.out'], cmd='exit 42')\n");

    build(true, GENRULE_ERROR, "//test:fail");

    BuildResultListener listener = getCommandEnvironment().getBuildResultListener();
    BuildResultPrinter printer = new BuildResultPrinter(getCommandEnvironment());
    printer.showBuildResult(
        getRequest(),
        getResult(),
        listener.getAnalyzedTargets(),
        listener.getSkippedTargets(),
        listener.getAnalyzedAspects(),
        listener.getTargetRootCauses(),
        listener.getAspectRootCauses(),
        /* hasSandboxedActionFailures= */ true);

    String stderr = recOutErr.errAsLatin1();
    assertThat(stderr)
        .contains(
            "Use --sandbox_debug to see verbose messages from the sandbox and retain the"
                + " sandbox build root for debugging\n");
  }

  @Test
  public void testIsSandboxedRunner() {
    assertThat(BuildResultListener.isSandboxedRunner("linux-sandbox")).isTrue();
    assertThat(BuildResultListener.isSandboxedRunner("docker")).isTrue();
    assertThat(BuildResultListener.isSandboxedRunner("standalone")).isFalse();
    assertThat(BuildResultListener.isSandboxedRunner(null)).isFalse();
  }

  // Concrete implementations of this abstract test:

  /** Tests with 1 job. */
  @RunWith(JUnit4.class)
  public static class SequentialBuildResultTest extends BuildResultTestCase {
    @Override
    protected int numJobs() {
      return 1;
    }
  }

  /** Tests with 100 jobs. */
  @RunWith(JUnit4.class)
  public static class ParallelBuildResultTest extends BuildResultTestCase {
    @Override
    protected int numJobs() { return 100; }
  }
}