Support setting different UI progress modes
This change adds a flag to select between UI progress modes. The initial
values are oldest_actions and mnemonic_histogram; the latter of these is
also added in this change.
The new mode turned out to be useful for debugging an issue where Bazel
ran out of memory due to a high number of ActionExecutionFunction
Skyframe restarts.
PiperOrigin-RevId: 255952215
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java
index 9a4bae2..06a3d50 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java
@@ -19,6 +19,7 @@
import com.google.devtools.build.lib.events.EventKind;
import com.google.devtools.build.lib.events.Location;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.runtime.ExperimentalStateTracker.ProgressMode;
import com.google.devtools.build.lib.util.io.OutErr;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.common.options.EnumConverter;
@@ -62,6 +63,13 @@
}
}
+ /** Progress mode converter. */
+ public static class ProgressModeConverter extends EnumConverter<ProgressMode> {
+ public ProgressModeConverter() {
+ super(ProgressMode.class, "--experimental_ui_mode setting");
+ }
+ }
+
public static class Options extends OptionsBase {
@Option(
@@ -223,17 +231,30 @@
public boolean experimentalUiDebugAllEvents;
@Option(
+ name = "experimental_ui_mode",
+ defaultValue = "oldest_actions",
+ converter = ProgressModeConverter.class,
+ documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+ effectTags = {OptionEffectTag.TERMINAL_OUTPUT},
+ help =
+ "Determines what kind of data is shown in the detailed progress bar. By default, it is "
+ + "set to show the oldest actions and their running time. The underlying data "
+ + "source is usually sampled in a mode-dependend way to fit within the number of "
+ + "lines given by --ui_actions_shown.")
+ public ProgressMode uiProgressMode;
+
+ @Option(
name = "ui_actions_shown",
oldName = "experimental_ui_actions_shown",
defaultValue = "8",
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
- effectTags = {OptionEffectTag.UNKNOWN},
+ effectTags = {OptionEffectTag.TERMINAL_OUTPUT},
help =
"Number of concurrent actions shown in the detailed progress bar; each "
+ "action is shown on a separate line. The progress bar always shows "
+ "at least one one, all numbers less than 1 are mapped to 1. "
+ "This option has no effect if --noui is set.")
- public int experimentalUiActionsShown;
+ public int uiSamplesShown;
@Option(
name = "experimental_ui_limit_console_output",
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/ExperimentalEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/ExperimentalEventHandler.java
index 9a048be..e6ffefe 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/ExperimentalEventHandler.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/ExperimentalEventHandler.java
@@ -270,7 +270,7 @@
this.cursorControl
? new ExperimentalStateTracker(clock, this.terminalWidth - 2)
: new ExperimentalStateTracker(clock);
- this.stateTracker.setSampleSize(options.experimentalUiActionsShown);
+ this.stateTracker.setProgressMode(options.uiProgressMode, options.uiSamplesShown);
this.numLinesProgressBar = 0;
if (this.cursorControl) {
this.minimalDelayMillis = Math.round(options.showProgressRateLimit * 1000);
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/ExperimentalStateTracker.java b/src/main/java/com/google/devtools/build/lib/runtime/ExperimentalStateTracker.java
index 32e572e..8d56640 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/ExperimentalStateTracker.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/ExperimentalStateTracker.java
@@ -18,7 +18,10 @@
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Comparators;
+import com.google.common.collect.HashMultiset;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Multiset;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionCompletionEvent;
@@ -57,6 +60,7 @@
import java.util.Comparator;
import java.util.Deque;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -69,6 +73,10 @@
* An experimental state tracker for the new experimental UI.
*/
class ExperimentalStateTracker {
+ enum ProgressMode {
+ OLDEST_ACTIONS,
+ MNEMONIC_HISTOGRAM
+ }
static final long SHOW_TIME_THRESHOLD_SECONDS = 3;
static final String ELLIPSIS = "...";
@@ -80,6 +88,7 @@
static final int NANOS_PER_SECOND = 1000000000;
static final String URL_PROTOCOL_SEP = "://";
+ private ProgressMode progressMode = ProgressMode.OLDEST_ACTIONS;
private int sampleSize = 3;
private String status;
@@ -342,15 +351,10 @@
this(clock, 0);
}
- /**
- * Set the maximal number of actions shown in the progress bar.
- */
- void setSampleSize(int sampleSize) {
- if (sampleSize >= 1) {
- this.sampleSize = sampleSize;
- } else {
- this.sampleSize = 1;
- }
+ /** Set the progress bar mode and sample size. */
+ void setProgressMode(ProgressMode progressMode, int sampleSize) {
+ this.progressMode = progressMode;
+ this.sampleSize = Math.max(1, sampleSize);
}
void buildStarted(BuildStartingEvent event) {
@@ -768,6 +772,31 @@
}
}
+ private void printActionState(AnsiTerminalWriter terminalWriter) throws IOException {
+ switch (progressMode) {
+ case OLDEST_ACTIONS:
+ sampleOldestActions(terminalWriter);
+ break;
+ case MNEMONIC_HISTOGRAM:
+ showMnemonicHistogram(terminalWriter);
+ break;
+ }
+ }
+
+ private void showMnemonicHistogram(AnsiTerminalWriter terminalWriter) throws IOException {
+ Multiset<String> mnemonicHistogram = HashMultiset.create();
+ for (Map.Entry<Artifact, ActionState> action : activeActions.entrySet()) {
+ mnemonicHistogram.add(action.getValue().action.getMnemonic());
+ }
+ List<Multiset.Entry<String>> sorted =
+ mnemonicHistogram.entrySet().stream()
+ .collect(
+ Comparators.greatest(sampleSize, Comparator.comparingLong((e) -> e.getCount())));
+ for (Multiset.Entry<String> entry : sorted) {
+ terminalWriter.newline().append(" " + entry.getElement() + " " + entry.getCount());
+ }
+ }
+
private void sampleOldestActions(AnsiTerminalWriter terminalWriter) throws IOException {
int count = 0;
int totalCount = 0;
@@ -1109,7 +1138,7 @@
terminalWriter.normal().append(" " + statusMessage);
maybeShowRecentTest(
terminalWriter, shortVersion, targetWidth - terminalWriter.getPosition());
- sampleOldestActions(terminalWriter);
+ printActionState(terminalWriter);
}
}
if (!shortVersion) {
diff --git a/src/test/java/com/google/devtools/build/lib/runtime/ExperimentalStateTrackerTest.java b/src/test/java/com/google/devtools/build/lib/runtime/ExperimentalStateTrackerTest.java
index dac992e..4176e9b 100644
--- a/src/test/java/com/google/devtools/build/lib/runtime/ExperimentalStateTrackerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/runtime/ExperimentalStateTrackerTest.java
@@ -46,6 +46,7 @@
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.events.ExtendedEventHandler.FetchProgress;
import com.google.devtools.build.lib.packages.AspectDescriptor;
+import com.google.devtools.build.lib.runtime.ExperimentalStateTracker.ProgressMode;
import com.google.devtools.build.lib.runtime.ExperimentalStateTracker.StrategyIds;
import com.google.devtools.build.lib.skyframe.LoadingPhaseStartedEvent;
import com.google.devtools.build.lib.skyframe.PackageProgressReceiver;
@@ -60,6 +61,7 @@
import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
import java.io.IOException;
import java.net.URL;
+import java.time.Duration;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -329,7 +331,7 @@
// For various sample sizes verify the progress bar
for (int i = 1; i < 11; i++) {
- stateTracker.setSampleSize(i);
+ stateTracker.setProgressMode(ProgressMode.OLDEST_ACTIONS, i);
LoggingTerminalWriter terminalWriter = new LoggingTerminalWriter(/*discardHighlight=*/ true);
stateTracker.writeProgressBar(terminalWriter);
String output = terminalWriter.getTranscript();
@@ -1248,6 +1250,43 @@
assertThat(output, containsString("30 fetches"));
}
+ private Action mockActionWithMnemonic(String mnemonic, String primaryOutput) {
+ Path path = outputBase.getRelative(PathFragment.create(primaryOutput));
+ Artifact artifact =
+ ActionsTestUtil.createArtifact(ArtifactRoot.asSourceRoot(Root.fromPath(outputBase)), path);
+ Action action = Mockito.mock(Action.class);
+ when(action.getMnemonic()).thenReturn(mnemonic);
+ when(action.getPrimaryOutput()).thenReturn(artifact);
+ return action;
+ }
+
+ @Test
+ public void testMnemonicHistogram() throws IOException {
+ // Verify that the number of actions shown in the progress bar can be set as sample size.
+ ManualClock clock = new ManualClock();
+ clock.advanceMillis(Duration.ofSeconds(123).toMillis());
+ ExperimentalStateTracker stateTracker = new ExperimentalStateTracker(clock);
+ clock.advanceMillis(Duration.ofSeconds(2).toMillis());
+
+ // Start actions with 10 different mnemonics Mnemonic0-9, n+1 of each mnemonic.
+ for (int i = 0; i < 10; i++) {
+ clock.advanceMillis(Duration.ofSeconds(1).toMillis());
+ for (int j = 0; j <= i; j++) {
+ Action action = mockActionWithMnemonic("Mnemonic" + i, "action-" + i + "-" + j + ".out");
+ stateTracker.actionStarted(new ActionStartedEvent(action, clock.nanoTime()));
+ }
+ }
+
+ for (int sampleSize = 1; sampleSize < 11; sampleSize++) {
+ stateTracker.setProgressMode(ProgressMode.MNEMONIC_HISTOGRAM, sampleSize);
+ LoggingTerminalWriter terminalWriter = new LoggingTerminalWriter(/*discardHighlight=*/ true);
+ stateTracker.writeProgressBar(terminalWriter);
+ String output = terminalWriter.getTranscript();
+ assertThat(output).contains("Mnemonic" + (10 - sampleSize) + " " + (10 - sampleSize + 1));
+ assertThat(output).doesNotContain("Mnemonic" + (10 - sampleSize - 1));
+ }
+ }
+
private static class FetchEvent implements FetchProgress {
private final String id;
