blob: cd75af0de538476d236966d0859ffa439308a639 [file] [log] [blame]
// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.buildtool;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.google.devtools.build.lib.skyframe.CoverageReportValue.COVERAGE_REPORT_KEY;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.actions.ActionCacheChecker;
import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
import com.google.devtools.build.lib.actions.ActionInputPrefetcher;
import com.google.devtools.build.lib.actions.ActionOutputDirectoryHelper;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.BuildFailedException;
import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.InputMetadataProvider;
import com.google.devtools.build.lib.actions.RemoteArtifactChecker;
import com.google.devtools.build.lib.actions.ResourceManager;
import com.google.devtools.build.lib.actions.TestExecException;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
import com.google.devtools.build.lib.analysis.test.TestProvider;
import com.google.devtools.build.lib.bugreport.BugReporter;
import com.google.devtools.build.lib.buildtool.buildevent.ExecutionProgressReceiverAvailableEvent;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.runtime.KeepGoingOption;
import com.google.devtools.build.lib.skyframe.ActionExecutionInactivityWatchdog;
import com.google.devtools.build.lib.skyframe.AspectKeyCreator.AspectKey;
import com.google.devtools.build.lib.skyframe.Builder;
import com.google.devtools.build.lib.skyframe.SkyframeErrorProcessor;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.DetailedExitCode;
import com.google.devtools.build.lib.util.DetailedExitCode.DetailedExitCodeComparator;
import com.google.devtools.build.lib.vfs.ModifiedFileSet;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.common.options.OptionsProvider;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
/**
* A {@link Builder} implementation driven by Skyframe.
*/
@VisibleForTesting
public class SkyframeBuilder implements Builder {
private final ResourceManager resourceManager;
private final SkyframeExecutor skyframeExecutor;
private final ModifiedFileSet modifiedOutputFiles;
private final InputMetadataProvider fileCache;
private final ActionInputPrefetcher actionInputPrefetcher;
private final ActionOutputDirectoryHelper actionOutputDirectoryHelper;
private final ActionCacheChecker actionCacheChecker;
private final BugReporter bugReporter;
@VisibleForTesting
public SkyframeBuilder(
SkyframeExecutor skyframeExecutor,
ResourceManager resourceManager,
ActionCacheChecker actionCacheChecker,
ModifiedFileSet modifiedOutputFiles,
InputMetadataProvider fileCache,
ActionInputPrefetcher actionInputPrefetcher,
ActionOutputDirectoryHelper actionOutputDirectoryHelper,
BugReporter bugReporter) {
this.resourceManager = resourceManager;
this.skyframeExecutor = skyframeExecutor;
this.actionCacheChecker = actionCacheChecker;
this.modifiedOutputFiles = modifiedOutputFiles;
this.fileCache = fileCache;
this.actionInputPrefetcher = actionInputPrefetcher;
this.actionOutputDirectoryHelper = actionOutputDirectoryHelper;
this.bugReporter = bugReporter;
}
@Override
public void buildArtifacts(
Reporter reporter,
Set<Artifact> artifacts,
Set<ConfiguredTarget> parallelTests,
Set<ConfiguredTarget> exclusiveTests,
Set<ConfiguredTarget> targetsToBuild,
Set<ConfiguredTarget> targetsToSkip,
ImmutableSet<AspectKey> aspects,
Executor executor,
OptionsProvider options,
@Nullable Range<Long> lastExecutionTimeRange,
TopLevelArtifactContext topLevelArtifactContext,
RemoteArtifactChecker remoteArtifactChecker)
throws BuildFailedException, AbruptExitException, TestExecException, InterruptedException {
BuildRequestOptions buildRequestOptions = options.getOptions(BuildRequestOptions.class);
// TODO(bazel-team): Should use --experimental_fsvc_threads instead of the hardcoded constant
// but plumbing the flag through is hard.
int fsvcThreads = buildRequestOptions == null ? 200 : buildRequestOptions.fsvcThreads;
skyframeExecutor.detectModifiedOutputFiles(
modifiedOutputFiles, lastExecutionTimeRange, remoteArtifactChecker, fsvcThreads);
try (SilentCloseable c = Profiler.instance().profile("configureActionExecutor")) {
skyframeExecutor.configureActionExecutor(fileCache, actionInputPrefetcher);
}
// Note that executionProgressReceiver accesses builtTargets concurrently (after wrapping in a
// synchronized collection), so unsynchronized access to this variable is unsafe while it runs.
ExecutionProgressReceiver executionProgressReceiver =
new ExecutionProgressReceiver(
countTestActions(exclusiveTests),
skyframeExecutor.getEventBus());
skyframeExecutor
.getEventBus()
.post(new ExecutionProgressReceiverAvailableEvent(executionProgressReceiver));
List<DetailedExitCode> detailedExitCodes = new ArrayList<>();
EvaluationResult<?> result;
ActionExecutionStatusReporter statusReporter = ActionExecutionStatusReporter.create(
reporter, skyframeExecutor.getEventBus());
AtomicBoolean isBuildingExclusiveArtifacts = new AtomicBoolean(false);
ActionExecutionInactivityWatchdog watchdog =
new ActionExecutionInactivityWatchdog(
executionProgressReceiver.createInactivityMonitor(statusReporter),
executionProgressReceiver.createInactivityReporter(
statusReporter, isBuildingExclusiveArtifacts),
options.getOptions(BuildRequestOptions.class).progressReportInterval);
skyframeExecutor.setActionExecutionProgressReportingObjects(executionProgressReceiver,
executionProgressReceiver, statusReporter);
watchdog.start();
// We need to extract out artifacts for the combined coverage report; these should only be built
// after any exclusive tests have been run, otherwise the tests get run as part of the build.
ImmutableSet<Artifact> coverageReportArtifacts =
artifacts.stream()
.filter(artifact -> artifact.getArtifactOwner().equals(COVERAGE_REPORT_KEY))
.collect(toImmutableSet());
Set<Artifact> artifactsToBuild = Sets.difference(artifacts, coverageReportArtifacts);
targetsToBuild = Sets.difference(targetsToBuild, targetsToSkip);
parallelTests = Sets.difference(parallelTests, targetsToSkip);
exclusiveTests = Sets.difference(exclusiveTests, targetsToSkip);
try {
result =
skyframeExecutor.buildArtifacts(
reporter,
resourceManager,
executor,
artifactsToBuild,
targetsToBuild,
aspects,
parallelTests,
exclusiveTests,
options,
actionCacheChecker,
actionOutputDirectoryHelper,
executionProgressReceiver,
topLevelArtifactContext);
// progressReceiver is finished, so unsynchronized access to builtTargets is now safe.
DetailedExitCode detailedExitCode =
SkyframeErrorProcessor.processResult(
reporter,
result,
options.getOptions(KeepGoingOption.class).keepGoing,
skyframeExecutor.getCyclesReporter(),
bugReporter);
if (detailedExitCode != null) {
detailedExitCodes.add(detailedExitCode);
}
// Run exclusive tests: either tagged as "exclusive" or is run in an invocation with
// --test_output=streamed.
isBuildingExclusiveArtifacts.set(true);
for (ConfiguredTarget exclusiveTest : exclusiveTests) {
// Since only one artifact is being built at a time, we don't worry about an artifact being
// built and then the build being interrupted.
result =
skyframeExecutor.runExclusiveTest(
reporter,
resourceManager,
executor,
exclusiveTest,
options,
actionCacheChecker,
actionOutputDirectoryHelper,
topLevelArtifactContext);
detailedExitCode =
SkyframeErrorProcessor.processResult(
reporter,
result,
options.getOptions(KeepGoingOption.class).keepGoing,
skyframeExecutor.getCyclesReporter(),
bugReporter);
Preconditions.checkState(
detailedExitCode != null || !result.keyNames().isEmpty(),
"Build reported as successful but test %s not executed: %s",
exclusiveTest,
result);
if (detailedExitCode != null) {
detailedExitCodes.add(detailedExitCode);
}
}
// Build coverage report
if (!coverageReportArtifacts.isEmpty()) {
result =
skyframeExecutor.evaluateSkyKeysWithExecution(
reporter,
executor,
Artifact.keys(coverageReportArtifacts),
options,
actionCacheChecker,
actionOutputDirectoryHelper);
detailedExitCode =
SkyframeErrorProcessor.processResult(
reporter,
result,
options.getOptions(KeepGoingOption.class).keepGoing,
skyframeExecutor.getCyclesReporter(),
bugReporter);
if (detailedExitCode != null) {
detailedExitCodes.add(detailedExitCode);
}
}
} finally {
watchdog.stop();
skyframeExecutor.setActionExecutionProgressReportingObjects(null, null, null);
statusReporter.unregisterFromEventBus();
}
if (detailedExitCodes.isEmpty()) {
return;
}
// Use the exit code with the highest priority.
throw new BuildFailedException(
null, Collections.max(detailedExitCodes, DetailedExitCodeComparator.INSTANCE));
}
ActionCacheChecker getActionCacheChecker() {
return actionCacheChecker;
}
InputMetadataProvider getFileCache() {
return fileCache;
}
ActionOutputDirectoryHelper getActionOutputDirectoryHelper() {
return actionOutputDirectoryHelper;
}
ActionInputPrefetcher getActionInputPrefetcher() {
return actionInputPrefetcher;
}
private static int countTestActions(Iterable<ConfiguredTarget> testTargets) {
int count = 0;
for (ConfiguredTarget testTarget : testTargets) {
count += TestProvider.getTestStatusArtifacts(testTarget).size();
}
return count;
}
}