src/test/java/com/google/devtools/build/lib/runtime/TestResultAggregatorTest.java

// Copyright 2015 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.runtime;

import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import com.google.common.collect.ImmutableList;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
import com.google.devtools.build.lib.analysis.test.TestProvider;
import com.google.devtools.build.lib.analysis.test.TestProvider.TestParams;
import com.google.devtools.build.lib.analysis.test.TestResult;
import com.google.devtools.build.lib.analysis.test.TestRunnerAction;
import com.google.devtools.build.lib.packages.TestTimeout;
import com.google.devtools.build.lib.runtime.TestResultAggregator.AggregationPolicy;
import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
import com.google.devtools.build.lib.view.test.TestStatus.TestResultData;
import java.util.stream.Stream;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

/** Tests for {@link TestResultAggregator}. */
@RunWith(JUnit4.class)
public final class TestResultAggregatorTest {

  private final TestParams mockParams = mock(TestParams.class);

  @Before
  public void configureMockParams() {
    when(mockParams.runsDetectsFlakes()).thenReturn(false);
    when(mockParams.getTimeout()).thenReturn(TestTimeout.LONG);
    when(mockParams.getShards()).thenReturn(1);
  }

  @Test
  public void nonCachedResult_setsActionRanTrue() {
    TestResultAggregator underTest = createAggregatorWithTestRuns(1);

    underTest.testEvent(
        testResult(TestResultData.newBuilder().setRemotelyCached(false), /*locallyCached=*/ false));

    assertThat(underTest.aggregateAndReportSummary(false).actionRan()).isTrue();
  }

  @Test
  public void locallyCachedTest_setsActionRanFalse() {
    TestResultAggregator underTest = createAggregatorWithTestRuns(1);

    underTest.testEvent(
        testResult(TestResultData.newBuilder().setRemotelyCached(false), /*locallyCached=*/ true));

    assertThat(underTest.aggregateAndReportSummary(false).actionRan()).isFalse();
  }

  @Test
  public void remotelyCachedTest_setsActionRanFalse() {
    TestResultAggregator underTest = createAggregatorWithTestRuns(1);

    underTest.testEvent(
        testResult(TestResultData.newBuilder().setRemotelyCached(true), /*locallyCached=*/ false));

    assertThat(underTest.aggregateAndReportSummary(false).actionRan()).isFalse();
  }

  @Test
  public void newCachedResult_keepsActionRanTrueWhenAlreadyTrue() {
    TestResultAggregator underTest = createAggregatorWithTestRuns(2);

    underTest.testEvent(
        testResult(TestResultData.newBuilder().setRemotelyCached(false), /*locallyCached=*/ false));
    underTest.testEvent(
        testResult(TestResultData.newBuilder().setRemotelyCached(true), /*locallyCached=*/ true));

    assertThat(underTest.aggregateAndReportSummary(false).actionRan()).isTrue();
  }

  @Test
  public void timingAggregation() {
    TestResultAggregator underTest = createAggregatorWithTestRuns(2);

    underTest.testEvent(
        testResult(
            TestResultData.newBuilder().setStartTimeMillisEpoch(7).setRunDurationMillis(10),
            /*locallyCached=*/ true));
    underTest.testEvent(
        testResult(
            TestResultData.newBuilder().setStartTimeMillisEpoch(12).setRunDurationMillis(1),
            /*locallyCached=*/ true));

    TestSummary summary = underTest.aggregateAndReportSummary(false);
    assertThat(summary.getFirstStartTimeMillis()).isEqualTo(7);
    assertThat(summary.getLastStopTimeMillis()).isEqualTo(17);
    assertThat(summary.getTotalRunDurationMillis()).isEqualTo(11);
  }

  @Test
  public void attemptCount_agggregatesSingleShardSingleAttempt() {
    when(mockParams.runsDetectsFlakes()).thenReturn(true);
    TestResultAggregator underTest = createAggregatorWithTestRuns(1);

    underTest.testEvent(
        shardedTestResult(
            TestResultData.newBuilder().addAllTestTimes(ImmutableList.of(1L, 2L)),
            /*shardNum=*/ 0));

    assertThat(underTest.aggregateAndReportSummary(false).getNumAttempts()).isEqualTo(2);
  }

  @Test
  public void attemptCount_agggregatesSingleShardMultipleAttempts() {
    when(mockParams.runsDetectsFlakes()).thenReturn(true);
    TestResultAggregator underTest = createAggregatorWithTestRuns(2);

    underTest.testEvent(
        shardedTestResult(
            TestResultData.newBuilder().addAllTestTimes(ImmutableList.of(1L, 2L)),
            /*shardNum=*/ 0));
    underTest.testEvent(
        shardedTestResult(
            TestResultData.newBuilder().addAllTestTimes(ImmutableList.of(3L, 4L)),
            /*shardNum=*/ 0));

    assertThat(underTest.aggregateAndReportSummary(false).getNumAttempts()).isEqualTo(4);
  }

  @Test
  public void attemptCount_agggregatesMultipleShardsMultipleAttempts() {
    when(mockParams.runsDetectsFlakes()).thenReturn(true);
    when(mockParams.getShards()).thenReturn(2);
    TestResultAggregator underTest = createAggregatorWithTestRuns(3);

    underTest.testEvent(
        shardedTestResult(
            TestResultData.newBuilder().addAllTestTimes(ImmutableList.of(1L, 2L, 3L)),
            /*shardNum=*/ 0));
    underTest.testEvent(
        shardedTestResult(
            TestResultData.newBuilder().addAllTestTimes(ImmutableList.of(3L, 4L)),
            /*shardNum=*/ 1));
    underTest.testEvent(
        shardedTestResult(
            TestResultData.newBuilder().addAllTestTimes(ImmutableList.of(3L, 4L)),
            /*shardNum=*/ 1));

    assertThat(underTest.aggregateAndReportSummary(false).getNumAttempts()).isEqualTo(4);
  }

  @Test
  public void attemptCount_agggregatesMultipleShardsSingleShardHasMostAttempts() {
    when(mockParams.runsDetectsFlakes()).thenReturn(true);
    when(mockParams.getShards()).thenReturn(2);
    TestResultAggregator underTest = createAggregatorWithTestRuns(3);

    underTest.testEvent(
        shardedTestResult(
            TestResultData.newBuilder().addAllTestTimes(ImmutableList.of(1L, 2L, 3L, 4L, 5L)),
            /*shardNum=*/ 0));
    underTest.testEvent(
        shardedTestResult(
            TestResultData.newBuilder().addAllTestTimes(ImmutableList.of(3L, 4L)),
            /*shardNum=*/ 1));
    underTest.testEvent(
        shardedTestResult(
            TestResultData.newBuilder().addAllTestTimes(ImmutableList.of(3L, 4L)),
            /*shardNum=*/ 1));

    assertThat(underTest.aggregateAndReportSummary(false).getNumAttempts()).isEqualTo(5);
  }

  @Test
  public void cancelConcurrentTests_cancellationAfterPassIgnored() {
    when(mockParams.runsDetectsFlakes()).thenReturn(true);
    TestResultAggregator underTest = createAggregatorWithTestRuns(2);

    underTest.testEvent(
        testResult(
            TestResultData.newBuilder().setStatus(BlazeTestStatus.PASSED),
            /*locallyCached=*/ true));
    underTest.testEvent(
        testResult(
            TestResultData.newBuilder().setStatus(BlazeTestStatus.INCOMPLETE),
            /*locallyCached=*/ true));

    assertThat(underTest.aggregateAndReportSummary(false).getStatus())
        .isEqualTo(BlazeTestStatus.PASSED);
  }

  @Test
  public void notAllTestRunsReported_skipTargetsOnFailure_noStatus() {
    TestResultAggregator underTest = createAggregatorWithTestRuns(2);

    underTest.testEvent(
        testResult(
            TestResultData.newBuilder().setStatus(BlazeTestStatus.PASSED),
            /*locallyCached=*/ false));

    assertThat(underTest.aggregateAndReportSummary(/*skipTargetsOnFailure=*/ true).getStatus())
        .isEqualTo(BlazeTestStatus.NO_STATUS);
  }

  @Test
  public void notAllTestRunsReported_noSkipTargetsOnFailure_incomplete() {
    TestResultAggregator underTest = createAggregatorWithTestRuns(2);

    underTest.testEvent(
        testResult(
            TestResultData.newBuilder().setStatus(BlazeTestStatus.PASSED),
            /*locallyCached=*/ false));

    assertThat(underTest.aggregateAndReportSummary(/*skipTargetsOnFailure=*/ false).getStatus())
        .isEqualTo(BlazeTestStatus.INCOMPLETE);
  }

  private TestResultAggregator createAggregatorWithTestRuns(int testRuns) {
    when(mockParams.getTestStatusArtifacts())
        .thenReturn(
            Stream.generate(() -> mock(DerivedArtifact.class))
                .limit(testRuns)
                .collect(toImmutableList()));
    when(mockParams.getRuns()).thenReturn(testRuns);

    ConfiguredTarget mockTarget = mock(ConfiguredTarget.class);
    when(mockTarget.getProvider(TestProvider.class)).thenReturn(new TestProvider(mockParams));

    return new TestResultAggregator(
        mockTarget,
        mock(BuildConfigurationValue.class),
        new AggregationPolicy(
            new EventBus(), /*testCheckUpToDate=*/ false, /*testVerboseTimeoutWarnings=*/ false),
        /*skippedThisTest=*/ false);
  }

  private static TestResult testResult(TestResultData.Builder data, boolean locallyCached) {
    TestRunnerAction mockTestAction = mock(TestRunnerAction.class);
    when(mockTestAction.getTestOutputsMapping(any(), any())).thenReturn(ImmutableList.of());
    return new TestResult(mockTestAction, data.build(), locallyCached, /*systemFailure=*/ null);
  }

  private static TestResult shardedTestResult(TestResultData.Builder data, int shardNum) {
    TestRunnerAction mockTestAction = mock(TestRunnerAction.class);
    when(mockTestAction.getTestOutputsMapping(any(), any())).thenReturn(ImmutableList.of());
    when(mockTestAction.getShardNum()).thenReturn(shardNum);
    return new TestResult(mockTestAction, data.build(), /*cached=*/ false, /*systemFailure=*/ null);
  }
}