src/test/java/com/google/devtools/build/lib/runtime/RetainedHeapLimiterTest.java

// Copyright 2019 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.runtime;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.verifyNoMoreInteractions;

import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.bugreport.BugReporter;
import com.google.devtools.build.lib.bugreport.Crash;
import com.google.devtools.build.lib.runtime.GcThrashingDetector.Limit;
import com.google.devtools.build.lib.runtime.MemoryPressure.MemoryPressureStats;
import com.google.devtools.build.lib.testutil.ManualClock;
import com.google.devtools.common.options.Options;
import com.google.testing.junit.testparameterinjector.TestParameterInjector;
import java.lang.ref.WeakReference;
import java.time.Duration;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.ArgumentMatchers;

/** Tests for {@link RetainedHeapLimiter}. */
@RunWith(TestParameterInjector.class)
public final class RetainedHeapLimiterTest {

  private final BugReporter bugReporter = mock(BugReporter.class);
  private final ManualClock clock = new ManualClock();
  private final MemoryPressureOptions options = Options.getDefaults(MemoryPressureOptions.class);

  private final RetainedHeapLimiter underTest =
      RetainedHeapLimiter.createForTest(bugReporter, clock);

  @Before
  public void setClock() {
    clock.advanceMillis(100000);
  }

  @After
  public void verifyNoMoreBugReports() {
    verifyNoMoreInteractions(bugReporter);
  }

  @Test
  public void underThreshold_noOom() {
    options.oomMoreEagerlyThreshold = 99;
    underTest.setOptions(options);

    underTest.handle(percentUsedAfterOrganicFullGc(100));
    underTest.handle(percentUsedAfterManualGc(89));

    verifyNoInteractions(bugReporter);
    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(1));
  }

  @Test
  public void overThreshold_oom() {
    options.oomMoreEagerlyThreshold = 90;
    underTest.setOptions(options);

    // Triggers GC, and tells RetainedHeapLimiter to OOM if too much memory used next time.
    underTest.handle(percentUsedAfterOrganicFullGc(91));

    underTest.handle(percentUsedAfterManualGc(91));

    ArgumentCaptor<Crash> crashArgument = ArgumentCaptor.forClass(Crash.class);
    verify(bugReporter).handleCrash(crashArgument.capture(), ArgumentMatchers.any());
    OutOfMemoryError oom = (OutOfMemoryError) crashArgument.getValue().getThrowable();
    assertThat(oom).hasMessageThat().contains("forcing exit due to GC thrashing");
    assertThat(oom).hasMessageThat().contains("tenured space is more than 90% occupied");

    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(1));
  }

  @Test
  public void inactiveAfterOom() {
    options.oomMoreEagerlyThreshold = 90;
    options.minTimeBetweenTriggeredGc = Duration.ZERO;
    underTest.setOptions(options);

    underTest.handle(percentUsedAfterOrganicFullGc(91));
    underTest.handle(percentUsedAfterManualGc(91));
    verify(bugReporter).handleCrash(any(), any());

    // No more GC or bug reports even if notifications come in after an OOM is in progress.
    WeakReference<?> ref = new WeakReference<>(new Object());
    clock.advanceMillis(Duration.ofMinutes(1).toMillis());
    underTest.handle(percentUsedAfterOrganicFullGc(91));
    underTest.handle(percentUsedAfterManualGc(91));
    assertThat(ref.get()).isNotNull();
    verifyNoMoreBugReports();

    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(1));
  }

  @Test
  public void externalGcNoTrigger() {
    options.oomMoreEagerlyThreshold = 90;
    underTest.setOptions(options);

    // No trigger because cause was "System.gc()".
    underTest.handle(percentUsedAfterManualGc(91));

    // Proof: no OOM.
    underTest.handle(percentUsedAfterManualGc(91));
    verifyNoInteractions(bugReporter);

    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(0));
  }

  @Test
  public void triggerReset() {
    options.oomMoreEagerlyThreshold = 90;
    underTest.setOptions(options);

    underTest.handle(percentUsedAfterOrganicFullGc(91));

    // Got under the threshold, so no OOM.
    underTest.handle(percentUsedAfterManualGc(89));

    // No OOM this time since wasn't triggered.
    underTest.handle(percentUsedAfterManualGc(91));
    verifyNoInteractions(bugReporter);
  }

  @Test
  public void triggerRaceWithOtherGc() {
    options.oomMoreEagerlyThreshold = 90;
    underTest.setOptions(options);

    underTest.handle(percentUsedAfterOrganicFullGc(91));
    underTest.handle(percentUsedAfterOrganicFullGc(91));
    underTest.handle(percentUsedAfterManualGc(91));

    ArgumentCaptor<Crash> crashArgument = ArgumentCaptor.forClass(Crash.class);
    verify(bugReporter).handleCrash(crashArgument.capture(), ArgumentMatchers.any());
    assertThat(crashArgument.getValue().getThrowable()).isInstanceOf(OutOfMemoryError.class);
  }

  @Test
  public void minTimeBetweenGc_lessThan_noGc() {
    options.oomMoreEagerlyThreshold = 90;
    options.minTimeBetweenTriggeredGc = Duration.ofMinutes(1);
    underTest.setOptions(options);
    WeakReference<?> ref = new WeakReference<>(new Object());

    underTest.handle(percentUsedAfterOrganicFullGc(91));
    assertThat(ref.get()).isNull();
    underTest.handle(percentUsedAfterManualGc(89));

    ref = new WeakReference<>(new Object());
    clock.advanceMillis(Duration.ofSeconds(59).toMillis());
    underTest.handle(percentUsedAfterOrganicFullGc(91));
    assertThat(ref.get()).isNotNull();

    assertStats(
        MemoryPressureStats.newBuilder()
            .setManuallyTriggeredGcs(1)
            .setMaxConsecutiveIgnoredGcsOverThreshold(1));
  }

  @Test
  public void minTimeBetweenGc_greaterThan_gc() {
    options.oomMoreEagerlyThreshold = 90;
    options.minTimeBetweenTriggeredGc = Duration.ofMinutes(1);
    underTest.setOptions(options);
    WeakReference<?> ref = new WeakReference<>(new Object());

    underTest.handle(percentUsedAfterOrganicFullGc(91));
    assertThat(ref.get()).isNull();
    underTest.handle(percentUsedAfterManualGc(89));

    ref = new WeakReference<>(new Object());
    clock.advanceMillis(Duration.ofSeconds(61).toMillis());
    underTest.handle(percentUsedAfterOrganicFullGc(91));
    assertThat(ref.get()).isNull();

    assertStats(
        MemoryPressureStats.newBuilder()
            .setManuallyTriggeredGcs(2)
            .setMaxConsecutiveIgnoredGcsOverThreshold(0));
  }

  @Test
  public void gcLockerDefersManualGc_timeoutCancelled() {
    options.oomMoreEagerlyThreshold = 90;
    options.minTimeBetweenTriggeredGc = Duration.ofMinutes(1);
    underTest.setOptions(options);

    underTest.handle(percentUsedAfterOrganicFullGc(91));
    WeakReference<?> ref = new WeakReference<>(new Object());
    underTest.handle(percentUsedAfterGcLockerGc(91));
    assertThat(ref.get()).isNull();

    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(2));
  }

  @Test
  public void gcLockerAfterSuccessfulManualGc_timeoutPreserved() {
    options.oomMoreEagerlyThreshold = 90;
    options.minTimeBetweenTriggeredGc = Duration.ofMinutes(1);
    underTest.setOptions(options);

    underTest.handle(percentUsedAfterOrganicFullGc(91));
    underTest.handle(percentUsedAfterManualGc(89));
    WeakReference<?> ref = new WeakReference<>(new Object());
    underTest.handle(percentUsedAfterGcLockerGc(91));
    assertThat(ref.get()).isNotNull();

    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(1));
  }

  @Test
  public void reportsMaxConsecutiveIgnored() {
    options.oomMoreEagerlyThreshold = 90;
    options.minTimeBetweenTriggeredGc = Duration.ofMinutes(1);
    underTest.setOptions(options);

    underTest.handle(percentUsedAfterOrganicFullGc(91));
    underTest.handle(percentUsedAfterManualGc(89));
    for (int i = 0; i < 6; i++) {
      underTest.handle(percentUsedAfterOrganicFullGc(91));
    }

    clock.advanceMillis(Duration.ofMinutes(2).toMillis());

    underTest.handle(percentUsedAfterOrganicFullGc(91));
    underTest.handle(percentUsedAfterManualGc(89));
    for (int i = 0; i < 8; i++) {
      underTest.handle(percentUsedAfterOrganicFullGc(91));
    }
    underTest.handle(percentUsedAfterOrganicFullGc(89)); // Breaks the streak of over threshold GCs.
    underTest.handle(percentUsedAfterOrganicFullGc(91));

    clock.advanceMillis(Duration.ofMinutes(2).toMillis());

    underTest.handle(percentUsedAfterOrganicFullGc(91));
    underTest.handle(percentUsedAfterOrganicFullGc(89));
    for (int i = 0; i < 7; i++) {
      underTest.handle(percentUsedAfterOrganicFullGc(91));
    }

    assertStats(
        MemoryPressureStats.newBuilder()
            .setManuallyTriggeredGcs(3)
            .setMaxConsecutiveIgnoredGcsOverThreshold(8));
  }

  @Test
  public void threshold100_noGcTriggeredEvenWithNonsenseStats() {
    options.oomMoreEagerlyThreshold = 100;
    underTest.setOptions(options);
    WeakReference<?> ref = new WeakReference<>(new Object());

    underTest.handle(percentUsedAfterOrganicFullGc(101));
    assertThat(ref.get()).isNotNull();

    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(0));
  }

  @Test
  public void optionsNotSet_disabled() {
    underTest.handle(percentUsedAfterOrganicFullGc(99));
    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(0));
  }

  @Test
  public void gcThrashingLimitsSet_mutuallyExclusive_disabled() {
    options.oomMoreEagerlyThreshold = 90;
    options.gcThrashingLimits = ImmutableList.of(Limit.of(Duration.ofMinutes(1), 2));
    options.gcThrashingLimitsRetainedHeapLimiterMutuallyExclusive = true;
    underTest.setOptions(options);

    underTest.handle(percentUsedAfterOrganicFullGc(99));

    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(0));
  }

  @Test
  public void gcThrashingLimitsSet_mutuallyInclusive_enabled() {
    options.oomMoreEagerlyThreshold = 90;
    options.gcThrashingLimits = ImmutableList.of(Limit.of(Duration.ofMinutes(1), 2));
    options.gcThrashingLimitsRetainedHeapLimiterMutuallyExclusive = false;
    underTest.setOptions(options);

    underTest.handle(percentUsedAfterOrganicFullGc(99));

    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(1));
  }

  @Test
  public void statsReset() {
    options.oomMoreEagerlyThreshold = 90;
    underTest.setOptions(options);

    underTest.handle(percentUsedAfterOrganicFullGc(91));

    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(1));
    assertStats(MemoryPressureStats.newBuilder().setManuallyTriggeredGcs(0));
  }

  private static MemoryPressureEvent percentUsedAfterManualGc(int percentUsed) {
    return percentUsedAfterGc(percentUsed).setWasManualGc(true).setWasFullGc(true).build();
  }

  private static MemoryPressureEvent percentUsedAfterOrganicFullGc(int percentUsed) {
    return percentUsedAfterGc(percentUsed).setWasFullGc(true).build();
  }

  private static MemoryPressureEvent percentUsedAfterGcLockerGc(int percentUsed) {
    return percentUsedAfterGc(percentUsed).setWasGcLockerInitiatedGc(true).build();
  }

  private static MemoryPressureEvent.Builder percentUsedAfterGc(int percentUsed) {
    checkArgument(percentUsed >= 0, percentUsed);
    return MemoryPressureEvent.newBuilder()
        .setTenuredSpaceUsedBytes(percentUsed)
        .setTenuredSpaceMaxBytes(100L);
  }

  private void assertStats(MemoryPressureStats.Builder expected) {
    MemoryPressureStats.Builder stats = MemoryPressureStats.newBuilder();
    underTest.addStatsAndReset(stats);
    assertThat(stats.build()).isEqualTo(expected.build());
  }
}