Record statistics about dirty output files detected in the output tree.
--
MOS_MIGRATED_REVID=88257621
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java
index 74143cc..9662e08 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java
@@ -23,6 +23,9 @@
* the metadata cache and about last file save times.
*/
public class ExecutionFinishedEvent {
+
+ private final int outputDirtyFiles;
+ private final int outputModifiedFilesDuringPreviousBuild;
/** The mtime of the most recently saved source file when the build starts. */
private long lastFileSaveTimeInMillis;
@@ -34,7 +37,10 @@
private Map<String, Long> changedFileSaveTimes = new HashMap<>();
public ExecutionFinishedEvent(Map<String, Long> changedFileSaveTimes,
- long lastFileSaveTimeInMillis) {
+ long lastFileSaveTimeInMillis, int outputDirtyFiles,
+ int outputModifiedFilesDuringPreviousBuild) {
+ this.outputDirtyFiles = outputDirtyFiles;
+ this.outputModifiedFilesDuringPreviousBuild = outputModifiedFilesDuringPreviousBuild;
this.changedFileSaveTimes = ImmutableMap.copyOf(changedFileSaveTimes);
this.lastFileSaveTimeInMillis = lastFileSaveTimeInMillis;
}
@@ -43,6 +49,14 @@
return lastFileSaveTimeInMillis;
}
+ public int getOutputDirtyFiles() {
+ return outputDirtyFiles;
+ }
+
+ public int getOutputModifiedFilesDuringPreviousBuild() {
+ return outputModifiedFilesDuringPreviousBuild;
+ }
+
public Map<String, Long> getChangedFileSaveTimes() {
return changedFileSaveTimes;
}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
index bd48395..d66da23 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -404,12 +404,14 @@
analysisResult.getExclusiveTests(),
analysisResult.getTargetsToBuild(),
executor, builtTargets,
- request.getBuildOptions().explanationPath != null);
+ request.getBuildOptions().explanationPath != null,
+ runtime.getLastExecutionTimeRange());
} catch (InterruptedException e) {
interrupted = true;
throw e;
} finally {
+ runtime.recordLastExecutionTime();
if (request.isRunningInEmacs()) {
request.getOutErr().printErrLn("blaze: Leaving directory `" + getExecRoot() + "/'");
}
@@ -417,8 +419,9 @@
getReporter().handle(Event.progress("Building complete."));
}
- // Transfer over source file "last save time" stats so the remote logger can find them.
- runtime.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap.<String, Long> of(), 0));
+ runtime.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap.<String, Long> of(), 0L,
+ skyframeExecutor.getOutputDirtyFiles(),
+ skyframeExecutor.getModifiedFilesDuringPreviousBuild()));
// Disable system load polling (noop if it was not enabled).
ResourceManager.instance().setAutoSensing(false);
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
index 779515a..c96c178 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
@@ -17,6 +17,7 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.Action;
@@ -91,9 +92,10 @@
Collection<ConfiguredTarget> targetsToBuild,
Executor executor,
Set<ConfiguredTarget> builtTargets,
- boolean explain)
+ boolean explain,
+ Range<Long> lastExecutionTimeRange)
throws BuildFailedException, AbruptExitException, TestExecException, InterruptedException {
- skyframeExecutor.prepareExecution(checkOutputFiles);
+ skyframeExecutor.prepareExecution(checkOutputFiles, lastExecutionTimeRange);
skyframeExecutor.setFileCache(fileCache);
// Note that executionProgressReceiver accesses builtTargets concurrently (after wrapping in a
// synchronized collection), so unsynchronized access to this variable is unsafe while it runs.
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
index d911820..b4308db 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
@@ -26,6 +26,7 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.SubscriberExceptionContext;
@@ -181,6 +182,8 @@
private Path workingDirectory;
private long commandStartTime;
+ private Range<Long> lastExecutionStartFinish = null;
+
private final SkyframeExecutor skyframeExecutor;
private final Reporter reporter;
@@ -418,6 +421,17 @@
}
}
+ public void recordLastExecutionTime() {
+ lastExecutionStartFinish = Range.closed(commandStartTime, clock.currentTimeMillis());
+ }
+
+ /**
+ * Range that represents the last execution time of a build in millis since epoch.
+ */
+ @Nullable
+ public Range<Long> getLastExecutionTimeRange() {
+ return lastExecutionStartFinish;
+ }
public void recordCommandStartTime(long commandStartTime) {
this.commandStartTime = commandStartTime;
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java
index 7fdb55c..f23b09f 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java
@@ -14,6 +14,7 @@
package com.google.devtools.build.lib.skyframe;
+import com.google.common.collect.Range;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.BuildFailedException;
import com.google.devtools.build.lib.actions.Executor;
@@ -25,6 +26,8 @@
import java.util.Collection;
import java.util.Set;
+import javax.annotation.Nullable;
+
/**
* A Builder consumes top-level artifacts, targets, and tests,, and executes them in some
* topological order, possibly concurrently, using some dependency-checking policy.
@@ -57,6 +60,8 @@
* @param builtTargets (out) set of successfully built subset of targetsToBuild. This set is
* populated immediately upon confirmation that artifact is built so it will be
* valid even if a future action throws ActionExecutionException
+ * @param lastExecutionTimeRange If not null, the start/finish time of the last build that
+ * run the execution phase.
* @throws BuildFailedException if there were problems establishing the action execution
* environment, if the the metadata of any file during the build could not be obtained,
* if any input files are missing, or if an action fails during execution
@@ -65,11 +70,12 @@
*/
@ThreadCompatible
void buildArtifacts(Set<Artifact> artifacts,
- Set<ConfiguredTarget> parallelTests,
- Set<ConfiguredTarget> exclusiveTests,
- Collection<ConfiguredTarget> targetsToBuild,
- Executor executor,
- Set<ConfiguredTarget> builtTargets,
- boolean explain)
+ Set<ConfiguredTarget> parallelTests,
+ Set<ConfiguredTarget> exclusiveTests,
+ Collection<ConfiguredTarget> targetsToBuild,
+ Executor executor,
+ Set<ConfiguredTarget> builtTargets,
+ boolean explain,
+ @Nullable Range<Long> lastExecutionTimeRange)
throws BuildFailedException, AbruptExitException, InterruptedException, TestExecException;
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java
index be4f4e8..d2e962a 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java
@@ -21,6 +21,7 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.devtools.build.lib.actions.Artifact;
@@ -71,11 +72,15 @@
SkyFunctionName.functionIs(SkyFunctions.ACTION_EXECUTION);
private final TimestampGranularityMonitor tsgm;
+ private final Range<Long> lastExecutionTimeRange;
private final Supplier<Map<SkyKey, SkyValue>> valuesSupplier;
private AtomicInteger modifiedOutputFilesCounter = new AtomicInteger(0);
+ private AtomicInteger modifiedOutputFilesIntraBuildCounter = new AtomicInteger(0);
- FilesystemValueChecker(final MemoizingEvaluator evaluator, TimestampGranularityMonitor tsgm) {
+ FilesystemValueChecker(final MemoizingEvaluator evaluator, TimestampGranularityMonitor tsgm,
+ Range<Long> lastExecutionTimeRange) {
this.tsgm = tsgm;
+ this.lastExecutionTimeRange = lastExecutionTimeRange;
// Construct the full map view of the entire graph at most once ("memoized"), lazily. If
// getDirtyFilesystemValues(Iterable<SkyKey>) is called on an empty Iterable, we avoid having
@@ -149,6 +154,7 @@
new ThrowableRecordingRunnableWrapper("FileSystemValueChecker#getDirtyActionValues");
modifiedOutputFilesCounter.set(0);
+ modifiedOutputFilesIntraBuildCounter.set(0);
for (List<Pair<SkyKey, ActionExecutionValue>> shard : outputShards) {
Runnable job = (batchStatter == null)
? outputStatJob(dirtyKeys, shard)
@@ -210,6 +216,7 @@
try {
FileValue newData = FileAndMetadataCache.fileValueFromArtifact(artifact, stat, tsgm);
if (!newData.equals(lastKnownData)) {
+ updateIntraBuildModifiedCounter(stat != null ? stat.getLastChangeTime() : -1);
modifiedOutputFilesCounter.getAndIncrement();
dirtyKeys.add(key);
}
@@ -223,6 +230,12 @@
};
}
+ private void updateIntraBuildModifiedCounter(long time) throws IOException {
+ if (lastExecutionTimeRange != null && lastExecutionTimeRange.contains(time)) {
+ modifiedOutputFilesIntraBuildCounter.incrementAndGet();
+ }
+ }
+
private Runnable outputStatJob(final Collection<SkyKey> dirtyKeys,
final List<Pair<SkyKey, ActionExecutionValue>> shard) {
return new Runnable() {
@@ -239,12 +252,19 @@
}
/**
- * Returns number of modified output files inside of dirty actions.
+ * Returns the number of modified output files inside of dirty actions.
*/
int getNumberOfModifiedOutputFiles() {
return modifiedOutputFilesCounter.get();
}
+ /**
+ * Returns the number of modified output files that occur during the previous build.
+ */
+ public int getNumberOfModifiedOutputFilesDuringPreviousBuild() {
+ return modifiedOutputFilesIntraBuildCounter.get();
+ }
+
private boolean actionValueIsDirtyWithDirectSystemCalls(ActionExecutionValue actionValue) {
boolean isDirty = false;
for (Map.Entry<Artifact, FileValue> entry :
@@ -252,8 +272,11 @@
Artifact artifact = entry.getKey();
FileValue lastKnownData = entry.getValue();
try {
- if (!FileAndMetadataCache.fileValueFromArtifact(artifact, null, tsgm).equals(
- lastKnownData)) {
+ FileValue fileValue = FileAndMetadataCache.fileValueFromArtifact(artifact, null, tsgm);
+ if (!fileValue.equals(lastKnownData)) {
+ updateIntraBuildModifiedCounter(fileValue.exists()
+ ? fileValue.realRootedPath().asPath().getLastModifiedTime()
+ : -1);
modifiedOutputFilesCounter.getAndIncrement();
isDirty = true;
}
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
index e547166..dc6dfac 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
@@ -323,7 +323,7 @@
// next evaluate() call), because checking those is a waste of time.
buildDriver.evaluate(ImmutableList.<SkyKey>of(), false,
DEFAULT_THREAD_COUNT, reporter);
- FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm);
+ FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm, null);
// We need to manually check for changes to known files. This entails finding all dirty file
// system values under package roots for which we don't have diff information. If at least
// one path entry doesn't have diff information, then we're going to have to iterate over
@@ -422,7 +422,7 @@
Iterable<SkyKey> keys;
if (modifiedFileSet.treatEverythingAsModified()) {
Differencer.Diff diff =
- new FilesystemValueChecker(memoizingEvaluator, tsgm).getDirtyFilesystemSkyKeys();
+ new FilesystemValueChecker(memoizingEvaluator, tsgm, null).getDirtyFilesystemSkyKeys();
keys = diff.changedKeysWithoutNewValues();
recordingDiffer.inject(diff.changedKeysWithNewValues());
} else {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
index 0c5b503..5bf3579 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -24,6 +24,7 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.Action;
@@ -214,6 +215,8 @@
private boolean needToInjectEmbeddedArtifacts = true;
private boolean needToInjectPrecomputedValuesForAnalysis = true;
protected int modifiedFiles;
+ protected int outputDirtyFiles;
+ protected int modifiedFilesDuringPreviousBuild;
private final Predicate<PathFragment> allowedMissingInputs;
private final ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions;
@@ -1416,15 +1419,19 @@
this.binTools = binTools;
}
- public void prepareExecution(boolean checkOutputFiles) throws AbruptExitException,
+ public void prepareExecution(boolean checkOutputFiles,
+ Range<Long> lastExecutionTimeRange) throws AbruptExitException,
InterruptedException {
maybeInjectEmbeddedArtifacts();
if (checkOutputFiles) {
// Detect external modifications in the output tree.
- FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm);
+ FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm,
+ lastExecutionTimeRange);
invalidateDirtyActions(fsnc.getDirtyActionValues(batchStatter));
modifiedFiles += fsnc.getNumberOfModifiedOutputFiles();
+ outputDirtyFiles += fsnc.getNumberOfModifiedOutputFiles();
+ modifiedFilesDuringPreviousBuild += fsnc.getNumberOfModifiedOutputFilesDuringPreviousBuild();
}
informAboutNumberOfModifiedFiles();
}
@@ -1519,4 +1526,11 @@
}
}
+ public int getOutputDirtyFiles() {
+ return outputDirtyFiles;
+ }
+
+ public int getModifiedFilesDuringPreviousBuild() {
+ return modifiedFilesDuringPreviousBuild;
+ }
}
