src/main/java/com/google/devtools/build/lib/runtime/RetainedHeapLimiter.java - bazel - Git at Google

 // Copyright 2016 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package com.google.devtools.build.lib.runtime;

 import static com.google.common.base.Preconditions.checkNotNull;

 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.flogger.GoogleLogger;
 import com.google.devtools.build.lib.bugreport.BugReporter;
 import com.google.devtools.build.lib.bugreport.Crash;
 import com.google.devtools.build.lib.bugreport.CrashContext;
 import com.google.devtools.build.lib.clock.BlazeClock;
 import com.google.devtools.build.lib.clock.Clock;
 import com.google.devtools.build.lib.concurrent.ThreadSafety;
 import com.google.devtools.build.lib.runtime.MemoryPressure.MemoryPressureStats;
 import com.google.devtools.common.options.Options;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;

 /**
  * Monitors the size of the retained heap and exit promptly if it grows too large.
  *
  * <p>Specifically, checks the size of the tenured space after each major GC; if it exceeds {@link
  * MemoryPressureOptions#oomMoreEagerlyThreshold}%, call {@link System#gc()} to trigger a
  * stop-the-world collection; if it's still more than {@link
  * MemoryPressureOptions#oomMoreEagerlyThreshold}% full, exit with an {@link OutOfMemoryError}.
  */
 final class RetainedHeapLimiter implements MemoryPressureStatCollector {

   private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();

   private final BugReporter bugReporter;
   private final Clock clock;

   private volatile MemoryPressureOptions options = inactiveOptions();

   private final AtomicBoolean throwingOom = new AtomicBoolean(false);
   private final AtomicBoolean heapLimiterTriggeredGc = new AtomicBoolean(false);
   private final AtomicInteger consecutiveIgnoredFullGcsOverThreshold = new AtomicInteger(0);
   private final AtomicBoolean loggedIgnoreWarningSinceLastGc = new AtomicBoolean(false);
   private final AtomicLong lastTriggeredGcMillis = new AtomicLong();
   private final AtomicInteger gcsTriggered = new AtomicInteger(0);
   private final AtomicInteger maxConsecutiveIgnoredFullGcsOverThreshold = new AtomicInteger(0);

   static RetainedHeapLimiter create(BugReporter bugReporter) {
     return new RetainedHeapLimiter(bugReporter, BlazeClock.instance());
   }

   @VisibleForTesting
   static RetainedHeapLimiter createForTest(BugReporter bugReporter, Clock clock) {
     return new RetainedHeapLimiter(bugReporter, clock);
   }

   private RetainedHeapLimiter(BugReporter bugReporter, Clock clock) {
     this.bugReporter = checkNotNull(bugReporter);
     this.clock = checkNotNull(clock);
   }

   @ThreadSafety.ThreadCompatible // Can only be called on the logical main Bazel thread.
   void setOptions(MemoryPressureOptions options) {
     if (options.gcThrashingLimitsRetainedHeapLimiterMutuallyExclusive
         && !options.gcThrashingLimits.isEmpty()) {
       this.options = inactiveOptions();
     } else {
       this.options = options;
     }
   }

   // Can be called concurrently, handles concurrent calls with #setThreshold gracefully.
   @ThreadSafety.ThreadSafe
   public void handle(MemoryPressureEvent event) {
     if (throwingOom.get()) {
       return; // Do nothing if a crash is already in progress.
     }

     boolean wasHeapLimiterTriggeredGc = false;
     boolean wasGcLockerDeferredHeapLimiterTriggeredGc = false;
     if (event.wasManualGc()) {
       wasHeapLimiterTriggeredGc = heapLimiterTriggeredGc.getAndSet(false);
       if (!wasHeapLimiterTriggeredGc) {
         // This was a manually triggered GC, but not from us earlier: short-circuit.
         logger.atInfo().log("Ignoring manual GC from other source");
         return;
       }
     } else if (event.wasGcLockerInitiatedGc() && heapLimiterTriggeredGc.getAndSet(false)) {
       // If System.gc() is called was while there are JNI thread(s) in the critical region, GCLocker
       // defers the GC until those threads exit the critical region. However, all GCLocker initiated
       // GCs are minor evacuation pauses, so we won't get the full GC we requested. Cancel the
       // timeout so we can attempt System.gc() again if we're still over the threshold. See full
       // explanation in b/263405096#comment14.
       logger.atWarning().log(
           "Observed a GCLocker initiated GC without observing a manual GC since the last call to"
               + " System.gc(), cancelling timeout to permit a retry");
       wasGcLockerDeferredHeapLimiterTriggeredGc = true;
       lastTriggeredGcMillis.set(0);
     }

     // Get a local reference to guard against concurrent modifications.
     MemoryPressureOptions options = this.options;
     int threshold = options.oomMoreEagerlyThreshold;

     if (threshold == 100) {
       return; // Inactive.
     }

     int actual = event.percentTenuredSpaceUsed();
     if (actual < threshold) {
       if (wasHeapLimiterTriggeredGc || wasGcLockerDeferredHeapLimiterTriggeredGc) {
         logger.atInfo().log("Back under threshold (%s%% of tenured space)", actual);
       }
       consecutiveIgnoredFullGcsOverThreshold.set(0);
       return;
     }

     if (wasHeapLimiterTriggeredGc) {
       if (!throwingOom.getAndSet(true)) {
         // We got here from a GC initiated by the other branch.
         OutOfMemoryError oom =
             new OutOfMemoryError(
                 String.format(
                     "RetainedHeapLimiter forcing exit due to GC thrashing: After back-to-back full"
                         + " GCs, the tenured space is more than %s%% occupied (%s out of a tenured"
                         + " space size of %s).",
                     threshold, event.tenuredSpaceUsedBytes(), event.tenuredSpaceMaxBytes()));
         logger.atInfo().log("Calling handleCrash");
         // Exits the runtime.
         bugReporter.handleCrash(Crash.from(oom), CrashContext.halt());
       }
     } else if (clock.currentTimeMillis() - lastTriggeredGcMillis.get()
         > options.minTimeBetweenTriggeredGc.toMillis()) {
       logger.atInfo().log(
           "Triggering a full GC (%s%% of tenured space after %s GC)",
           actual, event.wasFullGc() ? "full" : "minor");
       heapLimiterTriggeredGc.set(true);
       gcsTriggered.incrementAndGet();
       // Force a full stop-the-world GC and see if it can get us below the threshold.
       System.gc();
       lastTriggeredGcMillis.set(clock.currentTimeMillis());
       consecutiveIgnoredFullGcsOverThreshold.set(0);
       loggedIgnoreWarningSinceLastGc.set(false);
     } else if (event.wasFullGc()) {
       int consecutiveIgnored = consecutiveIgnoredFullGcsOverThreshold.incrementAndGet();
       maxConsecutiveIgnoredFullGcsOverThreshold.accumulateAndGet(consecutiveIgnored, Math::max);
       logger.atWarning().log(
           "Ignoring possible GC thrashing x%s (%s%% of tenured space after full GC) because of"
               + " recently triggered GC",
           consecutiveIgnored, actual);
     } else if (!loggedIgnoreWarningSinceLastGc.getAndSet(true)) {
       logger.atWarning().log(
           "Ignoring possible GC thrashing (%s%% of tenured space after minor GC) because of"
               + " recently triggered GC",
           actual);
     }
   }

   @Override
   public void addStatsAndReset(MemoryPressureStats.Builder stats) {
     stats
         .setManuallyTriggeredGcs(gcsTriggered.getAndSet(0))
         .setMaxConsecutiveIgnoredGcsOverThreshold(
             maxConsecutiveIgnoredFullGcsOverThreshold.getAndSet(0));
     consecutiveIgnoredFullGcsOverThreshold.set(0);
   }

   private static MemoryPressureOptions inactiveOptions() {
     var options = Options.getDefaults(MemoryPressureOptions.class);
     options.oomMoreEagerlyThreshold = 100;
     return options;
   }
 }
	// Copyright 2016 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package com.google.devtools.build.lib.runtime;

	import static com.google.common.base.Preconditions.checkNotNull;

	import com.google.common.annotations.VisibleForTesting;
	import com.google.common.flogger.GoogleLogger;
	import com.google.devtools.build.lib.bugreport.BugReporter;
	import com.google.devtools.build.lib.bugreport.Crash;
	import com.google.devtools.build.lib.bugreport.CrashContext;
	import com.google.devtools.build.lib.clock.BlazeClock;
	import com.google.devtools.build.lib.clock.Clock;
	import com.google.devtools.build.lib.concurrent.ThreadSafety;
	import com.google.devtools.build.lib.runtime.MemoryPressure.MemoryPressureStats;
	import com.google.devtools.common.options.Options;
	import java.util.concurrent.atomic.AtomicBoolean;
	import java.util.concurrent.atomic.AtomicInteger;
	import java.util.concurrent.atomic.AtomicLong;

	/**
	* Monitors the size of the retained heap and exit promptly if it grows too large.
	*
	* <p>Specifically, checks the size of the tenured space after each major GC; if it exceeds {@link
	* MemoryPressureOptions#oomMoreEagerlyThreshold}%, call {@link System#gc()} to trigger a
	* stop-the-world collection; if it's still more than {@link
	* MemoryPressureOptions#oomMoreEagerlyThreshold}% full, exit with an {@link OutOfMemoryError}.
	*/
	final class RetainedHeapLimiter implements MemoryPressureStatCollector {

	private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();

	private final BugReporter bugReporter;
	private final Clock clock;

	private volatile MemoryPressureOptions options = inactiveOptions();

	private final AtomicBoolean throwingOom = new AtomicBoolean(false);
	private final AtomicBoolean heapLimiterTriggeredGc = new AtomicBoolean(false);
	private final AtomicInteger consecutiveIgnoredFullGcsOverThreshold = new AtomicInteger(0);
	private final AtomicBoolean loggedIgnoreWarningSinceLastGc = new AtomicBoolean(false);
	private final AtomicLong lastTriggeredGcMillis = new AtomicLong();
	private final AtomicInteger gcsTriggered = new AtomicInteger(0);
	private final AtomicInteger maxConsecutiveIgnoredFullGcsOverThreshold = new AtomicInteger(0);

	static RetainedHeapLimiter create(BugReporter bugReporter) {
	return new RetainedHeapLimiter(bugReporter, BlazeClock.instance());
	}

	@VisibleForTesting
	static RetainedHeapLimiter createForTest(BugReporter bugReporter, Clock clock) {
	return new RetainedHeapLimiter(bugReporter, clock);
	}

	private RetainedHeapLimiter(BugReporter bugReporter, Clock clock) {
	this.bugReporter = checkNotNull(bugReporter);
	this.clock = checkNotNull(clock);
	}

	@ThreadSafety.ThreadCompatible // Can only be called on the logical main Bazel thread.
	void setOptions(MemoryPressureOptions options) {
	if (options.gcThrashingLimitsRetainedHeapLimiterMutuallyExclusive
	&& !options.gcThrashingLimits.isEmpty()) {
	this.options = inactiveOptions();
	} else {
	this.options = options;
	}
	}

	// Can be called concurrently, handles concurrent calls with #setThreshold gracefully.
	@ThreadSafety.ThreadSafe
	public void handle(MemoryPressureEvent event) {
	if (throwingOom.get()) {
	return; // Do nothing if a crash is already in progress.
	}

	boolean wasHeapLimiterTriggeredGc = false;
	boolean wasGcLockerDeferredHeapLimiterTriggeredGc = false;
	if (event.wasManualGc()) {
	wasHeapLimiterTriggeredGc = heapLimiterTriggeredGc.getAndSet(false);
	if (!wasHeapLimiterTriggeredGc) {
	// This was a manually triggered GC, but not from us earlier: short-circuit.
	logger.atInfo().log("Ignoring manual GC from other source");
	return;
	}
	} else if (event.wasGcLockerInitiatedGc() && heapLimiterTriggeredGc.getAndSet(false)) {
	// If System.gc() is called was while there are JNI thread(s) in the critical region, GCLocker
	// defers the GC until those threads exit the critical region. However, all GCLocker initiated
	// GCs are minor evacuation pauses, so we won't get the full GC we requested. Cancel the
	// timeout so we can attempt System.gc() again if we're still over the threshold. See full
	// explanation in b/263405096#comment14.
	logger.atWarning().log(
	"Observed a GCLocker initiated GC without observing a manual GC since the last call to"
	+ " System.gc(), cancelling timeout to permit a retry");
	wasGcLockerDeferredHeapLimiterTriggeredGc = true;
	lastTriggeredGcMillis.set(0);
	}

	// Get a local reference to guard against concurrent modifications.
	MemoryPressureOptions options = this.options;
	int threshold = options.oomMoreEagerlyThreshold;

	if (threshold == 100) {
	return; // Inactive.
	}

	int actual = event.percentTenuredSpaceUsed();
	if (actual < threshold) {
	if (wasHeapLimiterTriggeredGc \|\| wasGcLockerDeferredHeapLimiterTriggeredGc) {
	logger.atInfo().log("Back under threshold (%s%% of tenured space)", actual);
	}
	consecutiveIgnoredFullGcsOverThreshold.set(0);
	return;
	}

	if (wasHeapLimiterTriggeredGc) {
	if (!throwingOom.getAndSet(true)) {
	// We got here from a GC initiated by the other branch.
	OutOfMemoryError oom =
	new OutOfMemoryError(
	String.format(
	"RetainedHeapLimiter forcing exit due to GC thrashing: After back-to-back full"
	+ " GCs, the tenured space is more than %s%% occupied (%s out of a tenured"
	+ " space size of %s).",
	threshold, event.tenuredSpaceUsedBytes(), event.tenuredSpaceMaxBytes()));
	logger.atInfo().log("Calling handleCrash");
	// Exits the runtime.
	bugReporter.handleCrash(Crash.from(oom), CrashContext.halt());
	}
	} else if (clock.currentTimeMillis() - lastTriggeredGcMillis.get()
	> options.minTimeBetweenTriggeredGc.toMillis()) {
	logger.atInfo().log(
	"Triggering a full GC (%s%% of tenured space after %s GC)",
	actual, event.wasFullGc() ? "full" : "minor");
	heapLimiterTriggeredGc.set(true);
	gcsTriggered.incrementAndGet();
	// Force a full stop-the-world GC and see if it can get us below the threshold.
	System.gc();
	lastTriggeredGcMillis.set(clock.currentTimeMillis());
	consecutiveIgnoredFullGcsOverThreshold.set(0);
	loggedIgnoreWarningSinceLastGc.set(false);
	} else if (event.wasFullGc()) {
	int consecutiveIgnored = consecutiveIgnoredFullGcsOverThreshold.incrementAndGet();
	maxConsecutiveIgnoredFullGcsOverThreshold.accumulateAndGet(consecutiveIgnored, Math::max);
	logger.atWarning().log(
	"Ignoring possible GC thrashing x%s (%s%% of tenured space after full GC) because of"
	+ " recently triggered GC",
	consecutiveIgnored, actual);
	} else if (!loggedIgnoreWarningSinceLastGc.getAndSet(true)) {
	logger.atWarning().log(
	"Ignoring possible GC thrashing (%s%% of tenured space after minor GC) because of"
	+ " recently triggered GC",
	actual);
	}
	}

	@Override
	public void addStatsAndReset(MemoryPressureStats.Builder stats) {
	stats
	.setManuallyTriggeredGcs(gcsTriggered.getAndSet(0))
	.setMaxConsecutiveIgnoredGcsOverThreshold(
	maxConsecutiveIgnoredFullGcsOverThreshold.getAndSet(0));
	consecutiveIgnoredFullGcsOverThreshold.set(0);
	}

	private static MemoryPressureOptions inactiveOptions() {
	var options = Options.getDefaults(MemoryPressureOptions.class);
	options.oomMoreEagerlyThreshold = 100;
	return options;
	}
	}