Keep track of when memory pressure starts off in a bad state.
Previously we were only tracking if the memory pressure occurred during the build, but now if a build starts in a bad memory state we will get that recorded as well.
Also simplifies the ServerWatcher code.
Refactored testing notifications to work the same as the other anomalies where we register notification once and allow it to take different levels.
PiperOrigin-RevId: 420370411
diff --git a/src/main/java/com/google/devtools/build/lib/platform/SystemMemoryPressureEvent.java b/src/main/java/com/google/devtools/build/lib/platform/SystemMemoryPressureEvent.java
index 8b2a052..784e5fa 100644
--- a/src/main/java/com/google/devtools/build/lib/platform/SystemMemoryPressureEvent.java
+++ b/src/main/java/com/google/devtools/build/lib/platform/SystemMemoryPressureEvent.java
@@ -15,6 +15,7 @@
package com.google.devtools.build.lib.platform;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.devtools.build.lib.platform.SystemMemoryPressureMonitor.Level;
/**
* This event is fired from {@link
@@ -23,38 +24,10 @@
*/
public class SystemMemoryPressureEvent implements ExtendedEventHandler.Postable {
- /** The possible reasons a system could be suspended. */
- public enum Level {
- WARNING("Warning"),
- CRITICAL("Critical");
-
- private final String logString;
-
- Level(String logString) {
- this.logString = logString;
- }
-
- public String logString() {
- return logString;
- }
-
- /** These constants are mapped to enum in third_party/bazel/src/main/native/unix_jni.h. */
- static Level fromInt(int number) {
- switch (number) {
- case 0:
- return WARNING;
- case 1:
- return CRITICAL;
- default:
- throw new IllegalStateException("Unknown memory pressure level: " + number);
- }
- }
- };
-
private final Level level;
- public SystemMemoryPressureEvent(int level) {
- this.level = Level.fromInt(level);
+ public SystemMemoryPressureEvent(Level level) {
+ this.level = level;
}
public Level level() {
diff --git a/src/main/java/com/google/devtools/build/lib/platform/SystemMemoryPressureMonitor.java b/src/main/java/com/google/devtools/build/lib/platform/SystemMemoryPressureMonitor.java
index 11010cf..c3f7492 100644
--- a/src/main/java/com/google/devtools/build/lib/platform/SystemMemoryPressureMonitor.java
+++ b/src/main/java/com/google/devtools/build/lib/platform/SystemMemoryPressureMonitor.java
@@ -27,10 +27,8 @@
private static final SystemMemoryPressureMonitor singleton = new SystemMemoryPressureMonitor();
@GuardedBy("this")
- @Nullable private Reporter reporter;
-
- @GuardedBy("this")
- private int eventCount = 0;
+ @Nullable
+ private Reporter reporter;
public static SystemMemoryPressureMonitor getInstance() {
return singleton;
@@ -45,21 +43,57 @@
private native void registerJNI();
- /** The number of times that a memory pressure notification has been seen. */
- public synchronized int eventCount() {
- return eventCount;
+ private native int systemMemoryPressure();
+
+ /** The possible memory pressure levels. */
+ public enum Level {
+ NORMAL("Normal"),
+ WARNING("Warning"),
+ CRITICAL("Critical");
+
+ private final String logString;
+
+ Level(String logString) {
+ this.logString = logString;
+ }
+
+ public String logString() {
+ return logString;
+ }
+
+ /** These constants are mapped to enum in third_party/bazel/src/main/native/unix_jni.h. */
+ static Level fromInt(int number) {
+ switch (number) {
+ case 0:
+ return NORMAL;
+ case 1:
+ return WARNING;
+ case 2:
+ return CRITICAL;
+ default:
+ throw new IllegalStateException("Unknown memory pressure level: " + number);
+ }
+ }
+ };
+
+ /** Return current memory pressure */
+ public Level level() {
+ return Level.fromInt(systemMemoryPressure());
}
public synchronized void setReporter(@Nullable Reporter reporter) {
this.reporter = reporter;
+ int pressure = systemMemoryPressure();
+ if (Level.fromInt(pressure) != Level.NORMAL) {
+ memoryPressureCallback(pressure);
+ }
}
synchronized void memoryPressureCallback(int value) {
- SystemMemoryPressureEvent event = new SystemMemoryPressureEvent(value);
+ SystemMemoryPressureEvent event = new SystemMemoryPressureEvent(Level.fromInt(value));
if (reporter != null) {
reporter.post(event);
}
logger.atInfo().log("%s", event.logString());
- eventCount += 1;
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/server/ServerWatcherRunnable.java b/src/main/java/com/google/devtools/build/lib/server/ServerWatcherRunnable.java
index 998e758..69cff05 100644
--- a/src/main/java/com/google/devtools/build/lib/server/ServerWatcherRunnable.java
+++ b/src/main/java/com/google/devtools/build/lib/server/ServerWatcherRunnable.java
@@ -22,6 +22,7 @@
import com.google.common.flogger.GoogleLogger;
import com.google.devtools.build.lib.clock.BlazeClock;
import com.google.devtools.build.lib.platform.SystemMemoryPressureMonitor;
+import com.google.devtools.build.lib.platform.SystemMemoryPressureMonitor.Level;
import com.google.devtools.build.lib.unix.ProcMeminfoParser;
import com.google.devtools.build.lib.util.OS;
import io.grpc.Server;
@@ -85,28 +86,16 @@
}
/**
- * A low memory conditions checker that relies on memory pressure notifications.
+ * A low memory conditions checker that relies on memory pressure state.
*
- * <p>This checker will report a low memory condition when it detects a memory pressure
- * notification between the point when {@link #reset(long)} was called and {@link
- * #shouldShutdown()} is called.
- *
- * <p>Memory pressure notifications are provided by the platform-agnostic {@link
- * MemoryPressureCounter} class, which may be a no-op for the current platform.
+ * <p>Memory pressure state is provided by the platform-agnostic {@link
+ * SystemMemoryPressureMonitor} class, which may be a no-op for the current platform.
*/
private static class MemoryPressureLowMemoryChecker extends LowMemoryChecker {
- private final SystemMemoryPressureMonitor monitor = SystemMemoryPressureMonitor.getInstance();
- private int eventCountAtIdleStart = monitor.eventCount();
@Override
boolean check() {
- return monitor.eventCount() > eventCountAtIdleStart;
- }
-
- @Override
- void reset(long lastIdleTimeNanos) {
- super.reset(lastIdleTimeNanos);
- eventCountAtIdleStart = monitor.eventCount();
+ return SystemMemoryPressureMonitor.getInstance().level() != Level.NORMAL;
}
}
diff --git a/src/main/native/darwin/system_memory_pressure_jni.cc b/src/main/native/darwin/system_memory_pressure_jni.cc
index 6c6ce7e..8303586 100644
--- a/src/main/native/darwin/system_memory_pressure_jni.cc
+++ b/src/main/native/darwin/system_memory_pressure_jni.cc
@@ -20,6 +20,29 @@
namespace blaze_jni {
+static MemoryPressureLevel g_memory_pressure_level = MemoryPressureLevelNormal;
+
+static void call_memory_pressure_callback(
+ dispatch_source_memorypressure_flags_t pressure_level) {
+ if (pressure_level == DISPATCH_MEMORYPRESSURE_NORMAL) {
+ BAZEL_LOG(USER) << "memory pressure normal anomaly";
+ g_memory_pressure_level = MemoryPressureLevelNormal;
+ } else if (pressure_level == DISPATCH_MEMORYPRESSURE_WARN) {
+ BAZEL_LOG(USER) << "memory pressure warning anomaly";
+ g_memory_pressure_level = MemoryPressureLevelWarning;
+ } else if (pressure_level == DISPATCH_MEMORYPRESSURE_CRITICAL) {
+ BAZEL_LOG(USER) << "memory pressure critical anomaly";
+ g_memory_pressure_level = MemoryPressureLevelCritical;
+ } else {
+ BAZEL_LOG(FATAL) << "unknown memory pressure level: " << pressure_level;
+ }
+ memory_pressure_callback(g_memory_pressure_level);
+}
+
+MemoryPressureLevel portable_memory_pressure() {
+ return g_memory_pressure_level;
+}
+
void portable_start_memory_pressure_monitoring() {
// To test use:
// sudo memory_pressure -S -l warn
@@ -30,38 +53,45 @@
dispatch_queue_t queue = bazel::darwin::JniDispatchQueue();
dispatch_source_t source = dispatch_source_create(
DISPATCH_SOURCE_TYPE_MEMORYPRESSURE, 0,
- DISPATCH_MEMORYPRESSURE_WARN | DISPATCH_MEMORYPRESSURE_CRITICAL, queue);
+ DISPATCH_MEMORYPRESSURE_NORMAL | DISPATCH_MEMORYPRESSURE_WARN |
+ DISPATCH_MEMORYPRESSURE_CRITICAL,
+ queue);
BAZEL_CHECK_NE(source, nullptr);
dispatch_source_set_event_handler(source, ^{
- dispatch_source_memorypressure_flags_t pressureLevel =
+ dispatch_source_memorypressure_flags_t pressure_level =
dispatch_source_get_data(source);
- if (pressureLevel == DISPATCH_MEMORYPRESSURE_WARN) {
- BAZEL_LOG(USER) << "memory pressure warning anomaly";
- memory_pressure_callback(MemoryPressureLevelWarning);
- } else if (pressureLevel == DISPATCH_MEMORYPRESSURE_CRITICAL) {
- BAZEL_LOG(USER) << "memory pressure critical anomaly";
- memory_pressure_callback(MemoryPressureLevelCritical);
- } else {
- BAZEL_LOG(FATAL) << "unknown memory pressure critical level: "
- << pressureLevel;
- }
+ call_memory_pressure_callback(pressure_level);
});
dispatch_resume(source);
// These are registered solely so we can test the system from end-to-end.
// Using the Apple memory_pressure requires admin access.
int testToken;
int32_t status = notify_register_dispatch(
- "com.google.bazel.test.memorypressurelevel.warning", &testToken, queue,
- ^(int state) {
- BAZEL_LOG(USER) << "memory pressure test warning anomaly";
- memory_pressure_callback(MemoryPressureLevelWarning);
- });
- BAZEL_CHECK_EQ(status, NOTIFY_STATUS_OK);
- status = notify_register_dispatch(
- "com.google.bazel.test.memorypressurelevel.critical", &testToken, queue,
- ^(int state) {
- BAZEL_LOG(USER) << "memory pressure test critical anomaly";
- memory_pressure_callback(MemoryPressureLevelCritical);
+ "com.google.bazel.test.memorypressurelevel", &testToken, queue,
+ ^(int token) {
+ uint64_t state;
+ uint32_t status = notify_get_state(token, &state);
+ if (status != NOTIFY_STATUS_OK) {
+ BAZEL_LOG(FATAL) << "notify_get_state failed: " << status;
+ }
+ dispatch_source_memorypressure_flags_t pressure_level;
+ switch (state) {
+ case 0:
+ pressure_level = DISPATCH_MEMORYPRESSURE_NORMAL;
+ break;
+ case 1:
+ pressure_level = DISPATCH_MEMORYPRESSURE_WARN;
+ break;
+ case 2:
+ pressure_level = DISPATCH_MEMORYPRESSURE_CRITICAL;
+ break;
+ default:
+ BAZEL_LOG(FATAL)
+ << "unknown memory pressure level: " << state;
+ pressure_level = -1;
+ break;
+ }
+ call_memory_pressure_callback(pressure_level);
});
BAZEL_CHECK_EQ(status, NOTIFY_STATUS_OK);
});
diff --git a/src/main/native/unix_jni.cc b/src/main/native/unix_jni.cc
index 14a3cfb..09cc59a 100644
--- a/src/main/native/unix_jni.cc
+++ b/src/main/native/unix_jni.cc
@@ -1447,6 +1447,17 @@
}
}
+/*
+ * Class: Java_com_google_devtools_build_lib_platform_SystemMemoryPressureMonitor
+ * Method: systemMemoryPressure
+ * Signature: ()I
+ */
+extern "C" JNIEXPORT jint JNICALL
+Java_com_google_devtools_build_lib_platform_SystemMemoryPressureMonitor_systemMemoryPressure(
+ JNIEnv *env, jclass) {
+ return portable_memory_pressure();
+}
+
jobject g_disk_space_module;
/*
diff --git a/src/main/native/unix_jni.h b/src/main/native/unix_jni.h
index 1213d9f..8060e2d 100644
--- a/src/main/native/unix_jni.h
+++ b/src/main/native/unix_jni.h
@@ -142,14 +142,18 @@
// These need to be kept in sync with constants in
// j/c/g/devtools/build/lib/buildtool/buildevent/SystemMemoryPressureEvent.java
typedef enum {
- MemoryPressureLevelWarning = 0,
- MemoryPressureLevelCritical = 1,
+ MemoryPressureLevelNormal = 0,
+ MemoryPressureLevelWarning = 1,
+ MemoryPressureLevelCritical = 2,
} MemoryPressureLevel;
// Declaration for callback function that is called by memory pressure
// monitoring when memory pressure is detected.
extern void memory_pressure_callback(MemoryPressureLevel level);
+// Returns the current memory pressure.
+MemoryPressureLevel portable_memory_pressure();
+
// Starts up any infrastructure needed to do disk space monitoring.
// May be called more than once.
void portable_start_disk_space_monitoring();
diff --git a/src/main/native/unix_jni_bsd.cc b/src/main/native/unix_jni_bsd.cc
index 932867f..9f65854 100644
--- a/src/main/native/unix_jni_bsd.cc
+++ b/src/main/native/unix_jni_bsd.cc
@@ -152,6 +152,11 @@
// Currently not implemented.
}
+MemoryPressureLevel portable_memory_pressure() {
+ // Currently not implemented.
+ return MemoryPressureLevelNormal;
+}
+
void portable_start_disk_space_monitoring() {
// Currently not implemented.
}
diff --git a/src/main/native/unix_jni_linux.cc b/src/main/native/unix_jni_linux.cc
index 4855ab0..42221bc 100644
--- a/src/main/native/unix_jni_linux.cc
+++ b/src/main/native/unix_jni_linux.cc
@@ -127,6 +127,11 @@
// https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
}
+MemoryPressureLevel portable_memory_pressure() {
+ // Currently not implemented.
+ return MemoryPressureLevelNormal;
+}
+
void portable_start_disk_space_monitoring() {
// Currently not implemented.
}
diff --git a/src/main/native/windows/system_memory_pressure_jni.cc b/src/main/native/windows/system_memory_pressure_jni.cc
index 99fb88c..37417ee 100644
--- a/src/main/native/windows/system_memory_pressure_jni.cc
+++ b/src/main/native/windows/system_memory_pressure_jni.cc
@@ -31,3 +31,16 @@
// Currently not implemented.
// https://docs.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-creatememoryresourcenotification
}
+
+/*
+ * Class: Java_com_google_devtools_build_lib_platform_SystemMemoryPressureMonitor
+ * Method: systemMemoryPressure
+ * Signature: ()I
+ */
+extern "C" JNIEXPORT jint JNICALL
+Java_com_google_devtools_build_lib_platform_SystemMemoryPressureMonitor_systemMemoryPressure(
+ JNIEnv *env, jclass) {
+ // Currently not implemented.
+ return 0;
+}
+
diff --git a/src/test/java/com/google/devtools/build/lib/platform/SystemMemoryPressureEventTest.java b/src/test/java/com/google/devtools/build/lib/platform/SystemMemoryPressureEventTest.java
index 1ee0129b..51477c2 100644
--- a/src/test/java/com/google/devtools/build/lib/platform/SystemMemoryPressureEventTest.java
+++ b/src/test/java/com/google/devtools/build/lib/platform/SystemMemoryPressureEventTest.java
@@ -32,6 +32,7 @@
@RunWith(JUnit4.class)
public final class SystemMemoryPressureEventTest extends BuildIntegrationTestCase {
static class SystemMemoryPressureEventListener extends BlazeModule {
+ public int memoryPressureNormalEventCount = 0;
public int memoryPressureWarningEventCount = 0;
public int memoryPressureCriticalEventCount = 0;
@@ -43,6 +44,10 @@
@Subscribe
public void memoryPressureEvent(SystemMemoryPressureEvent event) {
switch (event.level()) {
+ case NORMAL:
+ ++memoryPressureNormalEventCount;
+ assertThat(event.logString()).isEqualTo("SystemMemoryPressureEvent: Normal");
+ break;
case WARNING:
++memoryPressureWarningEventCount;
assertThat(event.logString()).isEqualTo("SystemMemoryPressureEvent: Warning");
@@ -80,12 +85,14 @@
" outs = ['fire_memory_pressure_notifications.out'],",
" cmd = '"
+ notifierFilePath
- + " com.google.bazel.test.memorypressurelevel.warning 0 > $@; ' + ",
+ + " com.google.bazel.test.memorypressurelevel 0 > $@ && ' + ",
" '"
+ notifierFilePath
- + " com.google.bazel.test.memorypressurelevel.critical 0 >> $@',",
+ + " com.google.bazel.test.memorypressurelevel 1 >> $@ && ' + ",
+ " '" + notifierFilePath + " com.google.bazel.test.memorypressurelevel 2 >> $@',",
")");
buildTarget("//system_memory_pressure_event:fire_memory_pressure_notifications");
+ assertThat(eventListener.memoryPressureNormalEventCount).isGreaterThan(0);
assertThat(eventListener.memoryPressureWarningEventCount).isGreaterThan(0);
assertThat(eventListener.memoryPressureCriticalEventCount).isGreaterThan(0);
}
