Add number of registered / created actions to `buildMetrics` in the BEP. PiperOrigin-RevId: 660281783 Change-Id: I999634e2431823254ed5bbb469911af2dff4ec45
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseCompleteEvent.java index 6627b8c..85babc9 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseCompleteEvent.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisPhaseCompleteEvent.java
@@ -17,6 +17,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.TotalAndConfiguredTargetOnlyMetric; import com.google.devtools.build.lib.pkgcache.PackageManager.PackageManagerStatistics; import java.util.Collection; @@ -31,12 +32,14 @@ private final TotalAndConfiguredTargetOnlyMetric targetsConfigured; private final PackageManagerStatistics pkgManagerStats; private final TotalAndConfiguredTargetOnlyMetric actionsConstructed; + private final ImmutableMap<String, Integer> actionsConstructedByMnemonic; private final boolean analysisCacheDropped; public AnalysisPhaseCompleteEvent( Collection<? extends ConfiguredTarget> topLevelTargets, TotalAndConfiguredTargetOnlyMetric targetsConfigured, TotalAndConfiguredTargetOnlyMetric actionsConstructed, + ImmutableMap<String, Integer> actionsConstructedByMnemonic, long timeInMs, PackageManagerStatistics pkgManagerStats, boolean analysisCacheDropped) { @@ -45,6 +48,7 @@ this.targetsConfigured = checkNotNull(targetsConfigured); this.pkgManagerStats = pkgManagerStats; this.actionsConstructed = checkNotNull(actionsConstructed); + this.actionsConstructedByMnemonic = checkNotNull(actionsConstructedByMnemonic); this.analysisCacheDropped = analysisCacheDropped; } @@ -70,6 +74,10 @@ return actionsConstructed; } + public ImmutableMap<String, Integer> getActionsConstructedByMnemonic() { + return actionsConstructedByMnemonic; + } + public boolean wasAnalysisCacheDropped() { return analysisCacheDropped; }
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java index 9fa2cc8..0937a5e 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java
@@ -184,6 +184,10 @@ return skyframeBuildView.getEvaluatedActionCounts(); } + public ImmutableMap<String, Integer> getEvaluatedActionsCountsByMnemonic() { + return skyframeBuildView.getEvaluatedActionCountsByMnemonic(); + } + public PackageManagerStatistics getAndClearPkgManagerStatistics() { return skyframeExecutor.getPackageManager().getAndClearStatistics(); }
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto index fcefe36..88495b0 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto
@@ -933,6 +933,9 @@ // This is only set if all the actions reported a time google.protobuf.Duration system_time = 5; google.protobuf.Duration user_time = 6; + + // The total number of actions of this type registered during the build. + int64 actions_created = 7; } // Contains the top N actions by number of actions executed. repeated ActionData action_data = 4;
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisPhaseRunner.java b/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisPhaseRunner.java index 084afbc..4f3e104 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisPhaseRunner.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/AnalysisPhaseRunner.java
@@ -297,6 +297,7 @@ analysisResult.getTargetsToBuild(), view.getEvaluatedCounts(), view.getEvaluatedActionsCounts(), + view.getEvaluatedActionsCountsByMnemonic(), timer.stop().elapsed(TimeUnit.MILLISECONDS), view.getAndClearPkgManagerStatistics(), env.getSkyframeExecutor().wasAnalysisCacheInvalidatedAndResetBit()));
diff --git a/src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java b/src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java index 2a67f1a..8f513ee 100644 --- a/src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java +++ b/src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java
@@ -173,6 +173,13 @@ metrics.forEach(packageMetrics::addPackageLoadMetrics); } } + + ImmutableMap<String, Integer> actionsConstructedByMnemonic = + event.getActionsConstructedByMnemonic(); + for (var entry : actionsConstructedByMnemonic.entrySet()) { + ActionStats actionStats = actionStatsMap.computeIfAbsent(entry.getKey(), ActionStats::new); + actionStats.numActionsRegistered.addAndGet(entry.getValue()); + } } @SuppressWarnings("unused") @@ -229,7 +236,7 @@ public void onActionComplete(ActionCompletionEvent event) { ActionStats actionStats = actionStatsMap.computeIfAbsent(event.getAction().getMnemonic(), ActionStats::new); - actionStats.numActions.incrementAndGet(); + actionStats.numActionsExecuted.incrementAndGet(); actionStats.firstStarted.accumulate(event.getRelativeActionStartTimeNanos()); actionStats.lastEnded.accumulate(BlazeClock.nanoTime()); spawnStats.incrementActionCount(); @@ -355,7 +362,8 @@ actionStats.firstStarted.longValue())) .setLastEndedMs( nanosToMillisSinceEpochConverter.toEpochMillis(actionStats.lastEnded.longValue())) - .setActionsExecuted(actionStats.numActions.get()); + .setActionsExecuted(actionStats.numActionsExecuted.get()) + .setActionsCreated(actionStats.numActionsRegistered.get()); long systemTime = actionStats.systemTime.get(); if (systemTime > 0) { builder.setSystemTime(Durations.fromMillis(systemTime)); @@ -375,7 +383,7 @@ if (!recordMetricsForAllMnemonics) { actionStatsStream = actionStatsStream - .sorted(Comparator.comparingLong(a -> -a.numActions.get())) + .sorted(Comparator.comparingLong(a -> -a.numActionsExecuted.get())) .limit(MAX_ACTION_DATA); } @@ -582,7 +590,8 @@ final LongAccumulator firstStarted; final LongAccumulator lastEnded; - final AtomicLong numActions; + final AtomicLong numActionsExecuted; + final AtomicLong numActionsRegistered; final String mnemonic; final AtomicLong systemTime; final AtomicLong userTime; @@ -591,7 +600,8 @@ this.mnemonic = mnemonic; firstStarted = new LongAccumulator(Math::min, Long.MAX_VALUE); lastEnded = new LongAccumulator(Math::max, 0); - numActions = new AtomicLong(); + numActionsExecuted = new AtomicLong(); + numActionsRegistered = new AtomicLong(); systemTime = new AtomicLong(); userTime = new AtomicLong(); }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java index 0c4828b..819e3c6 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java
@@ -124,6 +124,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nullable; @@ -207,6 +208,15 @@ progressReceiver.actionCount.get(), progressReceiver.configuredTargetActionCount.get()); } + public ImmutableMap<String, Integer> getEvaluatedActionCountsByMnemonic() { + ImmutableMap.Builder<String, Integer> builder = ImmutableMap.builder(); + for (Map.Entry<String, AtomicInteger> entry : + progressReceiver.actionCountByMnemonic.entrySet()) { + builder.put(entry.getKey(), entry.getValue().get()); + } + return builder.buildOrThrow(); + } + /** * Returns a description of the analysis-cache affecting changes between the current configuration * and the incoming one. @@ -922,6 +932,7 @@ buildResultListener.getAnalyzedTargets(), getEvaluatedCounts(), getEvaluatedActionCounts(), + getEvaluatedActionCountsByMnemonic(), measuredAnalysisTime, skyframeExecutor.getPackageManager().getAndClearStatistics(), skyframeExecutor.wasAnalysisCacheInvalidatedAndResetBit())); @@ -1425,6 +1436,8 @@ private final AtomicInteger actionCount = new AtomicInteger(); private final AtomicInteger configuredTargetCount = new AtomicInteger(); private final AtomicInteger configuredTargetActionCount = new AtomicInteger(); + private final ConcurrentHashMap<String, AtomicInteger> actionCountByMnemonic = + new ConcurrentHashMap<>(); @Override public void dirtied(SkyKey skyKey, DirtyType dirtyType) { @@ -1467,10 +1480,17 @@ configuredTargetCount.incrementAndGet(); } configuredObjectCount.incrementAndGet(); - if (newValue instanceof ActionLookupValue) { + if (newValue instanceof ActionLookupValue alv) { // During multithreaded operation, this is only set to true, so no concurrency issues. someActionLookupValueEvaluated = true; - int numActions = ((ActionLookupValue) newValue).getNumActions(); + ImmutableList<ActionAnalysisMetadata> actions = alv.getActions(); + for (ActionAnalysisMetadata action : actions) { + actionCountByMnemonic + .computeIfAbsent(action.getMnemonic(), (m) -> new AtomicInteger(0)) + .incrementAndGet(); + } + + int numActions = actions.size(); actionCount.addAndGet(numActions); if (isConfiguredTarget) { configuredTargetActionCount.addAndGet(numActions);
diff --git a/src/test/java/com/google/devtools/build/lib/metrics/MetricsCollectorTest.java b/src/test/java/com/google/devtools/build/lib/metrics/MetricsCollectorTest.java index 4aecd5a..3d04824 100644 --- a/src/test/java/com/google/devtools/build/lib/metrics/MetricsCollectorTest.java +++ b/src/test/java/com/google/devtools/build/lib/metrics/MetricsCollectorTest.java
@@ -13,6 +13,7 @@ // limitations under the License. package com.google.devtools.build.lib.metrics; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; import static com.google.devtools.build.lib.testutil.TestConstants.PLATFORM_LABEL; @@ -135,6 +136,51 @@ } @Test + public void testActionsCreatedForIndividualMnemonics() throws Exception { + write( + "bar/BUILD", + """ + load(":bar.bzl", "bar") + bar(name = "bar") + """); + write( + "bar/bar.bzl", + """ + def _impl(ctx): + output1 = ctx.actions.declare_file(ctx.attr.name + ".out1") + output2 = ctx.actions.declare_file(ctx.attr.name + ".out2") + ctx.actions.write(output1, "foo") + ctx.actions.write(output2, "bar") + # Note that we created 2 actions, but pass along only one of their outputs. + return [DefaultInfo(files = depset([output1]))] + + bar = rule( + implementation = _impl, + ) + """); + buildTarget("//bar:bar"); + + BuildMetrics buildMetrics = buildMetricsEventListener.event.getBuildMetrics(); + List<ActionData> actionData = buildMetrics.getActionSummary().getActionDataList(); + ImmutableList<ActionData> fileWriteActions = + actionData.stream() + .filter(a -> a.getMnemonic().equals("FileWrite")) + .collect(toImmutableList()); + assertThat(fileWriteActions).hasSize(1); + ActionData fileWriteAction = fileWriteActions.get(0); + assertThat(fileWriteAction.getActionsCreated()).isEqualTo(2); + assertThat(fileWriteAction.getActionsExecuted()).isEqualTo(1); + + long totalActionsCreated = actionData.stream().mapToLong(ActionData::getActionsCreated).sum(); + long totalActionsExecuted = actionData.stream().mapToLong(ActionData::getActionsExecuted).sum(); + assertThat(totalActionsCreated).isEqualTo(3); + assertThat(totalActionsExecuted).isEqualTo(2); + assertThat(totalActionsCreated).isEqualTo(buildMetrics.getActionSummary().getActionsCreated()); + assertThat(totalActionsExecuted) + .isEqualTo(buildMetrics.getActionSummary().getActionsExecuted()); + } + + @Test public void buildGraphAndArtifactMetrics() throws Exception { write( "a/BUILD",