blob: b4dd11771877ac0bafbb6e2aa6539161f39c1c60 [file] [log] [blame]
Damien Martin-Guillerezf88f4d82015-09-25 13:56:55 +00001// Copyright 2014 The Bazel Authors. All rights reserved.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.buildtool;
15
16import com.google.common.base.Joiner;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010017import com.google.common.base.Predicate;
18import com.google.common.base.Stopwatch;
19import com.google.common.collect.HashBasedTable;
Philipp Wollermann45bf15b2015-06-18 13:06:16 +000020import com.google.common.collect.ImmutableList;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010021import com.google.common.collect.ImmutableMap;
22import com.google.common.collect.ImmutableSet;
23import com.google.common.collect.Iterables;
24import com.google.common.collect.Ordering;
25import com.google.common.collect.Table;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010026import com.google.devtools.build.lib.Constants;
27import com.google.devtools.build.lib.actions.Action;
28import com.google.devtools.build.lib.actions.ActionCacheChecker;
29import com.google.devtools.build.lib.actions.ActionContextConsumer;
30import com.google.devtools.build.lib.actions.ActionContextMarker;
31import com.google.devtools.build.lib.actions.ActionContextProvider;
32import com.google.devtools.build.lib.actions.ActionGraph;
33import com.google.devtools.build.lib.actions.ActionInputFileCache;
34import com.google.devtools.build.lib.actions.Artifact;
35import com.google.devtools.build.lib.actions.BlazeExecutor;
36import com.google.devtools.build.lib.actions.BuildFailedException;
37import com.google.devtools.build.lib.actions.ExecException;
38import com.google.devtools.build.lib.actions.ExecutionStrategy;
39import com.google.devtools.build.lib.actions.Executor;
40import com.google.devtools.build.lib.actions.Executor.ActionContext;
41import com.google.devtools.build.lib.actions.ExecutorInitException;
42import com.google.devtools.build.lib.actions.LocalHostCapacity;
43import com.google.devtools.build.lib.actions.ResourceManager;
Mark Schallerdffb6ee2015-02-25 20:01:01 +000044import com.google.devtools.build.lib.actions.ResourceSet;
Philipp Wollermann45bf15b2015-06-18 13:06:16 +000045import com.google.devtools.build.lib.actions.SimpleActionContextProvider;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010046import com.google.devtools.build.lib.actions.SpawnActionContext;
47import com.google.devtools.build.lib.actions.TestExecException;
48import com.google.devtools.build.lib.actions.cache.ActionCache;
49import com.google.devtools.build.lib.analysis.BuildView;
50import com.google.devtools.build.lib.analysis.BuildView.AnalysisResult;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010051import com.google.devtools.build.lib.analysis.ConfiguredTarget;
Philipp Wollermann45bf15b2015-06-18 13:06:16 +000052import com.google.devtools.build.lib.analysis.SymlinkTreeActionContext;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010053import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010054import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
55import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
56import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
57import com.google.devtools.build.lib.buildtool.buildevent.ExecutionPhaseCompleteEvent;
58import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010059import com.google.devtools.build.lib.events.Event;
60import com.google.devtools.build.lib.events.EventHandler;
61import com.google.devtools.build.lib.events.EventKind;
62import com.google.devtools.build.lib.events.Reporter;
63import com.google.devtools.build.lib.exec.CheckUpToDateFilter;
64import com.google.devtools.build.lib.exec.ExecutionOptions;
65import com.google.devtools.build.lib.exec.OutputService;
66import com.google.devtools.build.lib.exec.SingleBuildFileCache;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010067import com.google.devtools.build.lib.exec.SymlinkTreeStrategy;
Nathan Harmatad4803012015-09-08 20:03:22 +000068import com.google.devtools.build.lib.profiler.AutoProfiler;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010069import com.google.devtools.build.lib.profiler.ProfilePhase;
70import com.google.devtools.build.lib.profiler.Profiler;
71import com.google.devtools.build.lib.profiler.ProfilerTask;
72import com.google.devtools.build.lib.rules.fileset.FilesetActionContext;
73import com.google.devtools.build.lib.rules.fileset.FilesetActionContextImpl;
74import com.google.devtools.build.lib.rules.test.TestActionContext;
75import com.google.devtools.build.lib.runtime.BlazeModule;
76import com.google.devtools.build.lib.runtime.BlazeRuntime;
Ulf Adams633f5392015-09-15 11:13:08 +000077import com.google.devtools.build.lib.runtime.CommandEnvironment;
Dmitry Lomove2033b12015-08-19 16:57:49 +000078import com.google.devtools.build.lib.skyframe.AspectValue;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010079import com.google.devtools.build.lib.skyframe.Builder;
80import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010081import com.google.devtools.build.lib.util.AbruptExitException;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010082import com.google.devtools.build.lib.util.ExitCode;
83import com.google.devtools.build.lib.util.LoggingUtil;
Mark Schaller6df81792015-12-10 18:47:47 +000084import com.google.devtools.build.lib.util.Preconditions;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010085import com.google.devtools.build.lib.vfs.FileSystem;
86import com.google.devtools.build.lib.vfs.FileSystemUtils;
Eric Fellheimerf3b43af2015-11-13 19:51:20 +000087import com.google.devtools.build.lib.vfs.ModifiedFileSet;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010088import com.google.devtools.build.lib.vfs.Path;
89import com.google.devtools.build.lib.vfs.PathFragment;
90
91import java.io.IOException;
92import java.io.OutputStream;
93import java.io.PrintWriter;
94import java.util.ArrayList;
95import java.util.Collection;
96import java.util.HashMap;
97import java.util.HashSet;
98import java.util.LinkedHashSet;
99import java.util.List;
100import java.util.Map;
101import java.util.Set;
Philipp Wollermanna068dee2015-06-08 11:15:02 +0000102import java.util.TreeMap;
Eric Fellheimer2db6a742015-04-28 21:38:43 +0000103import java.util.UUID;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100104import java.util.concurrent.TimeUnit;
105import java.util.logging.Level;
106import java.util.logging.Logger;
107
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100108/**
109 * This class manages the execution phase. The entry point is {@link #executeBuild}.
110 *
111 * <p>This is only intended for use by {@link BuildTool}.
112 *
113 * <p>This class contains an ActionCache, and refers to the BlazeRuntime's BuildView and
114 * PackageCache.
115 *
116 * @see BuildTool
117 * @see BuildView
118 */
119public class ExecutionTool {
120 private static class StrategyConverter {
121 private Table<Class<? extends ActionContext>, String, ActionContext> classMap =
122 HashBasedTable.create();
123 private Map<Class<? extends ActionContext>, ActionContext> defaultClassMap =
124 new HashMap<>();
125
126 /**
127 * Aggregates all {@link ActionContext}s that are in {@code contextProviders}.
128 */
129 @SuppressWarnings("unchecked")
130 private StrategyConverter(Iterable<ActionContextProvider> contextProviders) {
131 for (ActionContextProvider provider : contextProviders) {
132 for (ActionContext strategy : provider.getActionContexts()) {
133 ExecutionStrategy annotation =
134 strategy.getClass().getAnnotation(ExecutionStrategy.class);
135 if (annotation != null) {
136 defaultClassMap.put(annotation.contextType(), strategy);
137
138 for (String name : annotation.name()) {
139 classMap.put(annotation.contextType(), name, strategy);
140 }
141 }
142 }
143 }
144 }
145
146 @SuppressWarnings("unchecked")
147 private <T extends ActionContext> T getStrategy(Class<T> clazz, String name) {
148 return (T) (name.isEmpty() ? defaultClassMap.get(clazz) : classMap.get(clazz, name));
149 }
150
151 private String getValidValues(Class<? extends ActionContext> context) {
152 return Joiner.on(", ").join(Ordering.natural().sortedCopy(classMap.row(context).keySet()));
153 }
154
155 private String getUserFriendlyName(Class<? extends ActionContext> context) {
156 ActionContextMarker marker = context.getAnnotation(ActionContextMarker.class);
157 return marker != null
158 ? marker.name()
159 : context.getSimpleName();
160 }
161 }
162
163 static final Logger LOG = Logger.getLogger(ExecutionTool.class.getName());
164
Ulf Adams633f5392015-09-15 11:13:08 +0000165 private final CommandEnvironment env;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100166 private final BlazeRuntime runtime;
167 private final BuildRequest request;
168 private BlazeExecutor executor;
169 private ActionInputFileCache fileCache;
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000170 private final ImmutableList<ActionContextProvider> actionContextProviders;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100171
Philipp Wollermanna068dee2015-06-08 11:15:02 +0000172 private Map<String, SpawnActionContext> spawnStrategyMap =
173 new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100174 private List<ActionContext> strategies = new ArrayList<>();
175
Ulf Adams633f5392015-09-15 11:13:08 +0000176 ExecutionTool(CommandEnvironment env, BuildRequest request) throws ExecutorInitException {
177 this.env = env;
178 this.runtime = env.getRuntime();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100179 this.request = request;
180
Philipp Wollermann590ea392015-08-25 13:57:33 +0000181 // Create tools before getting the strategies from the modules as some of them need tools to
182 // determine whether the host actually supports certain strategies (e.g. sandboxing).
183 createToolsSymlinks();
184
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000185 this.actionContextProviders =
186 getActionContextProvidersFromModules(
Ulf Adams633f5392015-09-15 11:13:08 +0000187 runtime,
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000188 new FilesetActionContextImpl.Provider(
Ulf Adams633f5392015-09-15 11:13:08 +0000189 env.getReporter(), runtime.getWorkspaceName()),
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000190 new SimpleActionContextProvider(
Ulf Adams706b7f22015-10-20 09:06:45 +0000191 new SymlinkTreeStrategy(env.getOutputService(), runtime.getBinTools())));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100192 StrategyConverter strategyConverter = new StrategyConverter(actionContextProviders);
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000193
194 ImmutableList<ActionContextConsumer> actionContextConsumers =
195 getActionContextConsumersFromModules(
Ulf Adams633f5392015-09-15 11:13:08 +0000196 runtime,
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000197 // TODO(philwo) - the ExecutionTool should not add arbitrary dependencies on its own,
198 // instead these dependencies should be added to the ActionContextConsumer of the module
199 // that actually depends on them.
200 new ActionContextConsumer() {
201 @Override
202 public Map<String, String> getSpawnActionContexts() {
203 return ImmutableMap.of();
204 }
205
206 @Override
207 public Map<Class<? extends ActionContext>, String> getActionContexts() {
208 return ImmutableMap.<Class<? extends ActionContext>, String>builder()
209 .put(FilesetActionContext.class, "")
210 .put(WorkspaceStatusAction.Context.class, "")
211 .put(SymlinkTreeActionContext.class, "")
212 .build();
213 }
214 });
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100215
216 for (ActionContextConsumer consumer : actionContextConsumers) {
217 // There are many different SpawnActions, and we want to control the action context they use
218 // independently from each other, for example, to run genrules locally and Java compile action
219 // in prod. Thus, for SpawnActions, we decide the action context to use not only based on the
220 // context class, but also the mnemonic of the action.
221 for (Map.Entry<String, String> entry : consumer.getSpawnActionContexts().entrySet()) {
222 SpawnActionContext context =
223 strategyConverter.getStrategy(SpawnActionContext.class, entry.getValue());
224 if (context == null) {
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000225 throw makeExceptionForInvalidStrategyValue(
226 entry.getValue(),
227 "spawn",
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100228 strategyConverter.getValidValues(SpawnActionContext.class));
229 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100230 spawnStrategyMap.put(entry.getKey(), context);
231 }
232
233 for (Map.Entry<Class<? extends ActionContext>, String> entry :
234 consumer.getActionContexts().entrySet()) {
235 ActionContext context = strategyConverter.getStrategy(entry.getKey(), entry.getValue());
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000236 if (context == null) {
237 throw makeExceptionForInvalidStrategyValue(
238 entry.getValue(),
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100239 strategyConverter.getUserFriendlyName(entry.getKey()),
240 strategyConverter.getValidValues(entry.getKey()));
241 }
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000242 strategies.add(context);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100243 }
244 }
245
246 // If tests are to be run during build, too, we have to explicitly load the test action context.
247 if (request.shouldRunTests()) {
248 String testStrategyValue = request.getOptions(ExecutionOptions.class).testStrategy;
249 ActionContext context = strategyConverter.getStrategy(TestActionContext.class,
250 testStrategyValue);
251 if (context == null) {
252 throw makeExceptionForInvalidStrategyValue(testStrategyValue, "test",
253 strategyConverter.getValidValues(TestActionContext.class));
254 }
255 strategies.add(context);
256 }
257 }
258
Ulf Adams633f5392015-09-15 11:13:08 +0000259 private static ImmutableList<ActionContextConsumer> getActionContextConsumersFromModules(
260 BlazeRuntime runtime, ActionContextConsumer... extraConsumers) {
261 ImmutableList.Builder<ActionContextConsumer> builder = ImmutableList.builder();
262 for (BlazeModule module : runtime.getBlazeModules()) {
263 builder.addAll(module.getActionContextConsumers());
264 }
265 builder.add(extraConsumers);
266 return builder.build();
267 }
268
269 private static ImmutableList<ActionContextProvider> getActionContextProvidersFromModules(
270 BlazeRuntime runtime, ActionContextProvider... extraProviders) {
271 ImmutableList.Builder<ActionContextProvider> builder = ImmutableList.builder();
272 for (BlazeModule module : runtime.getBlazeModules()) {
273 builder.addAll(module.getActionContextProviders());
274 }
275 builder.add(extraProviders);
276 return builder.build();
277 }
278
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100279 private static ExecutorInitException makeExceptionForInvalidStrategyValue(String value,
280 String strategy, String validValues) {
281 return new ExecutorInitException(String.format(
282 "'%s' is an invalid value for %s strategy. Valid values are: %s", value, strategy,
283 validValues), ExitCode.COMMAND_LINE_ERROR);
284 }
285
286 Executor getExecutor() throws ExecutorInitException {
287 if (executor == null) {
288 executor = createExecutor();
289 }
290 return executor;
291 }
292
293 /**
294 * Creates an executor for the current set of blaze runtime, execution options, and request.
295 */
296 private BlazeExecutor createExecutor()
297 throws ExecutorInitException {
298 return new BlazeExecutor(
299 runtime.getDirectories().getExecRoot(),
300 runtime.getDirectories().getOutputPath(),
301 getReporter(),
Ulf Adams633f5392015-09-15 11:13:08 +0000302 env.getEventBus(),
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100303 runtime.getClock(),
304 request,
305 request.getOptions(ExecutionOptions.class).verboseFailures,
306 request.getOptions(ExecutionOptions.class).showSubcommands,
307 strategies,
308 spawnStrategyMap,
309 actionContextProviders);
310 }
311
312 void init() throws ExecutorInitException {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100313 getExecutor();
314 }
315
316 void shutdown() {
317 for (ActionContextProvider actionContextProvider : actionContextProviders) {
318 actionContextProvider.executionPhaseEnding();
319 }
320 }
321
322 /**
323 * Performs the execution phase (phase 3) of the build, in which the Builder
324 * is applied to the action graph to bring the targets up to date. (This
325 * function will return prior to execution-proper if --nobuild was specified.)
Eric Fellheimer2db6a742015-04-28 21:38:43 +0000326 * @param buildId UUID of the build id
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100327 * @param analysisResult the analysis phase output
328 * @param buildResult the mutable build result
Lukacs Berki3ea4d442016-01-21 15:15:30 +0000329 * @param packageRoots package roots collected from loading phase and BuildConfigurationCollection
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100330 * creation
331 */
Eric Fellheimer2db6a742015-04-28 21:38:43 +0000332 void executeBuild(UUID buildId, AnalysisResult analysisResult,
Ulf Adams633f5392015-09-15 11:13:08 +0000333 BuildResult buildResult,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100334 BuildConfigurationCollection configurations,
335 ImmutableMap<PathFragment, Path> packageRoots)
Nathan Harmataca06fa22015-07-27 19:54:14 +0000336 throws BuildFailedException, InterruptedException, TestExecException, AbruptExitException {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100337 Stopwatch timer = Stopwatch.createStarted();
338 prepare(packageRoots, configurations);
339
340 ActionGraph actionGraph = analysisResult.getActionGraph();
341
342 // Get top-level artifacts.
343 ImmutableSet<Artifact> additionalArtifacts = analysisResult.getAdditionalArtifactsToBuild();
344
Ulf Adams706b7f22015-10-20 09:06:45 +0000345 OutputService outputService = env.getOutputService();
Eric Fellheimerf3b43af2015-11-13 19:51:20 +0000346 ModifiedFileSet modifiedOutputFiles = ModifiedFileSet.EVERYTHING_MODIFIED;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100347 if (outputService != null) {
Eric Fellheimer1246eb32015-11-18 15:41:38 +0000348 modifiedOutputFiles = outputService.startBuild(buildId,
349 request.getBuildOptions().finalizeActions);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100350 } else {
351 startLocalOutputBuild(); // TODO(bazel-team): this could be just another OutputService
352 }
353
Lukacs Berki85c63c42016-01-22 09:25:20 +0000354 if (getWorkspace().getFileSystem().supportsSymbolicLinks()) {
355 List<BuildConfiguration> targetConfigurations = configurations.getTargetConfigurations();
356 BuildConfiguration targetConfiguration = targetConfigurations.size() == 1
357 ? targetConfigurations.get(0) : null;
358 if (targetConfigurations.size() == 1) {
359 OutputDirectoryLinksUtils.createOutputDirectoryLinks(
360 runtime.getWorkspaceName(), getWorkspace(), getExecRoot(),
361 runtime.getOutputPath(), getReporter(), targetConfiguration,
362 request.getBuildOptions().getSymlinkPrefix());
363 }
364 }
365
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100366 ActionCache actionCache = getActionCache();
Ulf Adams80613022015-09-16 09:11:33 +0000367 SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
Eric Fellheimerf3b43af2015-11-13 19:51:20 +0000368 Builder builder =
369 createBuilder(request, executor, actionCache, skyframeExecutor, modifiedOutputFiles);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100370
371 //
372 // Execution proper. All statements below are logically nested in
373 // begin/end pairs. No early returns or exceptions please!
374 //
375
376 Collection<ConfiguredTarget> configuredTargets = buildResult.getActualTargets();
Ulf Adams633f5392015-09-15 11:13:08 +0000377 env.getEventBus().post(new ExecutionStartingEvent(configuredTargets));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100378
379 getReporter().handle(Event.progress("Building..."));
380
381 // Conditionally record dependency-checker log:
382 ExplanationHandler explanationHandler =
383 installExplanationHandler(request.getBuildOptions().explanationPath,
384 request.getOptionsDescription());
385
386 Set<ConfiguredTarget> builtTargets = new HashSet<>();
Janak Ramakrishnana5d8d092015-12-08 18:32:47 +0000387 Collection<AspectValue> aspects = analysisResult.getAspects();
388
389 Iterable<Artifact> allArtifactsForProviders =
390 Iterables.concat(
391 additionalArtifacts,
392 TopLevelArtifactHelper.getAllArtifactsToBuild(
393 analysisResult.getTargetsToBuild(), analysisResult.getTopLevelContext())
394 .getAllArtifacts(),
395 TopLevelArtifactHelper.getAllArtifactsToBuildFromAspects(
396 aspects, analysisResult.getTopLevelContext())
397 .getAllArtifacts(),
398 //TODO(dslomov): Artifacts to test from aspects?
399 TopLevelArtifactHelper.getAllArtifactsToTest(analysisResult.getTargetsToTest()));
400
401 if (request.isRunningInEmacs()) {
402 // The syntax of this message is tightly constrained by lisp/progmodes/compile.el in emacs
403 request.getOutErr().printErrLn("blaze: Entering directory `" + getExecRoot() + "/'");
404 }
405 boolean buildCompleted = false;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100406 try {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100407 for (ActionContextProvider actionContextProvider : actionContextProviders) {
408 actionContextProvider.executionPhaseStarting(
409 fileCache,
410 actionGraph,
411 allArtifactsForProviders);
412 }
413 executor.executionPhaseStarting();
414 skyframeExecutor.drainChangedFiles();
415
416 if (request.getViewOptions().discardAnalysisCache) {
417 // Free memory by removing cache entries that aren't going to be needed. Note that in
418 // skyframe full, this destroys the action graph as well, so we can only do it after the
419 // action graph is no longer needed.
Ulf Adamsea44dcb2015-09-18 12:36:30 +0000420 env.getView().clearAnalysisCache(analysisResult.getTargetsToBuild());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100421 actionGraph = null;
422 }
423
424 configureResourceManager(request);
425
426 Profiler.instance().markPhase(ProfilePhase.EXECUTE);
427
Dmitry Lomove2033b12015-08-19 16:57:49 +0000428 builder.buildArtifacts(
Ulf Adams5b9009b2015-09-24 09:52:53 +0000429 env.getReporter(),
Dmitry Lomove2033b12015-08-19 16:57:49 +0000430 additionalArtifacts,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100431 analysisResult.getParallelTests(),
432 analysisResult.getExclusiveTests(),
433 analysisResult.getTargetsToBuild(),
Dmitry Lomove2033b12015-08-19 16:57:49 +0000434 analysisResult.getAspects(),
435 executor,
436 builtTargets,
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +0000437 request.getBuildOptions().explanationPath != null,
438 runtime.getLastExecutionTimeRange());
Janak Ramakrishnana5d8d092015-12-08 18:32:47 +0000439 buildCompleted = true;
440 } catch (BuildFailedException | TestExecException e) {
441 buildCompleted = true;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100442 throw e;
443 } finally {
Ulf Adamsb5146102015-10-20 08:57:26 +0000444 env.recordLastExecutionTime();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100445 if (request.isRunningInEmacs()) {
446 request.getOutErr().printErrLn("blaze: Leaving directory `" + getExecRoot() + "/'");
447 }
Janak Ramakrishnana5d8d092015-12-08 18:32:47 +0000448 if (buildCompleted) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100449 getReporter().handle(Event.progress("Building complete."));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100450 }
451
Ulf Adams633f5392015-09-15 11:13:08 +0000452 env.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap.<String, Long> of(), 0L,
Miguel Alcon Pinto9186f682015-04-09 21:49:34 +0000453 skyframeExecutor.getOutputDirtyFilesAndClear(),
454 skyframeExecutor.getModifiedFilesDuringPreviousBuildAndClear()));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100455
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100456 executor.executionPhaseEnding();
457 for (ActionContextProvider actionContextProvider : actionContextProviders) {
458 actionContextProvider.executionPhaseEnding();
459 }
460
461 Profiler.instance().markPhase(ProfilePhase.FINISH);
462
Janak Ramakrishnana5d8d092015-12-08 18:32:47 +0000463 if (buildCompleted) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100464 saveCaches(actionCache);
465 }
466
Nathan Harmatad4803012015-09-08 20:03:22 +0000467 try (AutoProfiler p = AutoProfiler.profiled("Show results", ProfilerTask.INFO)) {
Ulf Adams7a3fe522015-12-04 10:48:56 +0000468 buildResult.setSuccessfulTargets(
469 determineSuccessfulTargets(configuredTargets, builtTargets, timer));
Googler3ca7b782015-10-12 15:44:09 +0000470 BuildResultPrinter buildResultPrinter = new BuildResultPrinter(env);
471 buildResultPrinter.showBuildResult(
472 request, buildResult, configuredTargets, analysisResult.getAspects());
Nathan Harmatad4803012015-09-08 20:03:22 +0000473 }
Googler3ca7b782015-10-12 15:44:09 +0000474
475 try (AutoProfiler p = AutoProfiler.profiled("Show artifacts", ProfilerTask.INFO)) {
476 if (request.getBuildOptions().showArtifacts) {
477 BuildResultPrinter buildResultPrinter = new BuildResultPrinter(env);
478 buildResultPrinter.showArtifacts(
479 request, configuredTargets, analysisResult.getAspects());
480 }
481 }
482
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100483 if (explanationHandler != null) {
484 uninstallExplanationHandler(explanationHandler);
485 }
486 // Finalize output service last, so that if we do throw an exception, we know all the other
487 // code has already run.
Ulf Adams706b7f22015-10-20 09:06:45 +0000488 if (env.getOutputService() != null) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100489 boolean isBuildSuccessful =
490 buildResult.getSuccessfulTargets().size() == configuredTargets.size();
Ulf Adams706b7f22015-10-20 09:06:45 +0000491 env.getOutputService().finalizeBuild(isBuildSuccessful);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100492 }
493 }
494 }
495
496 private void prepare(ImmutableMap<PathFragment, Path> packageRoots,
Nathan Harmataca06fa22015-07-27 19:54:14 +0000497 BuildConfigurationCollection configurations) throws ExecutorInitException {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100498 // Prepare for build.
499 Profiler.instance().markPhase(ProfilePhase.PREPARE);
500
501 // Create some tools symlinks / cleanup per-build state
502 createActionLogDirectory();
503
504 // Plant the symlink forest.
505 plantSymlinkForest(packageRoots, configurations);
506 }
507
508 private void createToolsSymlinks() throws ExecutorInitException {
509 try {
510 runtime.getBinTools().setupBuildTools();
511 } catch (ExecException e) {
Nathan Harmataca06fa22015-07-27 19:54:14 +0000512 throw new ExecutorInitException("Tools symlink creation failed", e);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100513 }
514 }
515
516 private void plantSymlinkForest(ImmutableMap<PathFragment, Path> packageRoots,
Nathan Harmataca06fa22015-07-27 19:54:14 +0000517 BuildConfigurationCollection configurations) throws ExecutorInitException {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100518 try {
519 FileSystemUtils.deleteTreesBelowNotPrefixed(getExecRoot(),
520 new String[] { ".", "_", Constants.PRODUCT_NAME + "-"});
521 // Delete the build configuration's temporary directories
522 for (BuildConfiguration configuration : configurations.getTargetConfigurations()) {
523 configuration.prepareForExecutionPhase();
524 }
525 FileSystemUtils.plantLinkForest(packageRoots, getExecRoot());
526 } catch (IOException e) {
Nathan Harmataca06fa22015-07-27 19:54:14 +0000527 throw new ExecutorInitException("Source forest creation failed", e);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100528 }
529 }
530
Nathan Harmataca06fa22015-07-27 19:54:14 +0000531 private void createActionLogDirectory() throws ExecutorInitException {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100532 Path directory = runtime.getDirectories().getActionConsoleOutputDirectory();
533 try {
534 if (directory.exists()) {
535 FileSystemUtils.deleteTree(directory);
536 }
537 directory.createDirectory();
Nathan Harmataca06fa22015-07-27 19:54:14 +0000538 } catch (IOException e) {
539 throw new ExecutorInitException("Couldn't delete action output directory", e);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100540 }
541 }
542
543 /**
544 * Prepare for a local output build.
545 */
Nathan Harmataca06fa22015-07-27 19:54:14 +0000546 private void startLocalOutputBuild() throws ExecutorInitException {
Nathan Harmatad4803012015-09-08 20:03:22 +0000547 try (AutoProfiler p = AutoProfiler.profiled("Starting local output build", ProfilerTask.INFO)) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100548 Path outputPath = runtime.getOutputPath();
549 Path localOutputPath = runtime.getDirectories().getLocalOutputPath();
550
551 if (outputPath.isSymbolicLink()) {
Nathan Harmataca06fa22015-07-27 19:54:14 +0000552 try {
553 // Remove the existing symlink first.
554 outputPath.delete();
555 if (localOutputPath.exists()) {
556 // Pre-existing local output directory. Move to outputPath.
557 localOutputPath.renameTo(outputPath);
558 }
559 } catch (IOException e) {
560 throw new ExecutorInitException("Couldn't handle local output directory symlinks", e);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100561 }
562 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100563 }
564 }
565
566 /**
567 * If a path is supplied, creates and installs an ExplanationHandler. Returns
568 * an instance on success. Reports an error and returns null otherwise.
569 */
570 private ExplanationHandler installExplanationHandler(PathFragment explanationPath,
571 String allOptions) {
572 if (explanationPath == null) {
573 return null;
574 }
575 ExplanationHandler handler;
576 try {
577 handler = new ExplanationHandler(
578 getWorkspace().getRelative(explanationPath).getOutputStream(),
579 allOptions);
580 } catch (IOException e) {
581 getReporter().handle(Event.warn(String.format(
582 "Cannot write explanation of rebuilds to file '%s': %s",
583 explanationPath, e.getMessage())));
584 return null;
585 }
586 getReporter().handle(
587 Event.info("Writing explanation of rebuilds to '" + explanationPath + "'"));
588 getReporter().addHandler(handler);
589 return handler;
590 }
591
592 /**
593 * Uninstalls the specified ExplanationHandler (if any) and closes the log
594 * file.
595 */
596 private void uninstallExplanationHandler(ExplanationHandler handler) {
597 if (handler != null) {
598 getReporter().removeHandler(handler);
599 handler.log.close();
600 }
601 }
602
603 /**
604 * An ErrorEventListener implementation that records DEPCHECKER events into a log
605 * file, iff the --explain flag is specified during a build.
606 */
607 private static class ExplanationHandler implements EventHandler {
608
609 private final PrintWriter log;
610
611 private ExplanationHandler(OutputStream log, String optionsDescription) {
612 this.log = new PrintWriter(log);
613 this.log.println("Build options: " + optionsDescription);
614 }
615
616
617 @Override
618 public void handle(Event event) {
619 if (event.getKind() == EventKind.DEPCHECKER) {
620 log.println(event.getMessage());
621 }
622 }
623 }
624
625 /**
626 * Computes the result of the build. Sets the list of successful (up-to-date)
627 * targets in the request object.
628 *
629 * @param configuredTargets The configured targets whose artifacts are to be
630 * built.
631 * @param timer A timer that was started when the execution phase started.
632 */
Ulf Adams7a3fe522015-12-04 10:48:56 +0000633 private Collection<ConfiguredTarget> determineSuccessfulTargets(
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100634 Collection<ConfiguredTarget> configuredTargets, Set<ConfiguredTarget> builtTargets,
635 Stopwatch timer) {
636 // Maintain the ordering by copying builtTargets into a LinkedHashSet in the same iteration
637 // order as configuredTargets.
638 Collection<ConfiguredTarget> successfulTargets = new LinkedHashSet<>();
639 for (ConfiguredTarget target : configuredTargets) {
640 if (builtTargets.contains(target)) {
641 successfulTargets.add(target);
642 }
643 }
Ulf Adams633f5392015-09-15 11:13:08 +0000644 env.getEventBus().post(
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100645 new ExecutionPhaseCompleteEvent(timer.stop().elapsed(TimeUnit.MILLISECONDS)));
Ulf Adams7a3fe522015-12-04 10:48:56 +0000646 return successfulTargets;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100647 }
648
Dmitry Lomov56c7d552015-09-07 14:57:47 +0000649
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100650 private ActionCache getActionCache() throws LocalEnvironmentException {
651 try {
Ulf Adams80613022015-09-16 09:11:33 +0000652 return env.getPersistentActionCache();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100653 } catch (IOException e) {
654 // TODO(bazel-team): (2010) Ideally we should just remove all cache data and reinitialize
655 // caches.
656 LoggingUtil.logToRemote(Level.WARNING, "Failed to initialize action cache: "
657 + e.getMessage(), e);
658 throw new LocalEnvironmentException("couldn't create action cache: " + e.getMessage()
659 + ". If error persists, use 'blaze clean'");
660 }
661 }
662
663 private Builder createBuilder(BuildRequest request,
664 Executor executor,
665 ActionCache actionCache,
Eric Fellheimerf3b43af2015-11-13 19:51:20 +0000666 SkyframeExecutor skyframeExecutor,
667 ModifiedFileSet modifiedOutputFiles) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100668 BuildRequest.BuildRequestOptions options = request.getBuildOptions();
669 boolean verboseExplanations = options.verboseExplanations;
670 boolean keepGoing = request.getViewOptions().keepGoing;
671
672 Path actionOutputRoot = runtime.getDirectories().getActionConsoleOutputDirectory();
673 Predicate<Action> executionFilter = CheckUpToDateFilter.fromOptions(
674 request.getOptions(ExecutionOptions.class));
675
676 // jobs should have been verified in BuildRequest#validateOptions().
677 Preconditions.checkState(options.jobs >= -1);
678 int actualJobs = options.jobs == 0 ? 1 : options.jobs; // Treat 0 jobs as a single task.
679
680 // Unfortunately, the exec root cache is not shared with caches in the remote execution
681 // client.
682 fileCache = createBuildSingleFileCache(executor.getExecRoot());
683 skyframeExecutor.setActionOutputRoot(actionOutputRoot);
684 return new SkyframeBuilder(skyframeExecutor,
Ulf Adamsea44dcb2015-09-18 12:36:30 +0000685 new ActionCacheChecker(actionCache, env.getView().getArtifactFactory(), executionFilter,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100686 verboseExplanations),
Eric Fellheimerf3b43af2015-11-13 19:51:20 +0000687 keepGoing, actualJobs,
688 options.checkOutputFiles ? modifiedOutputFiles : ModifiedFileSet.NOTHING_MODIFIED,
689 options.finalizeActions, fileCache, request.getBuildOptions().progressReportInterval);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100690 }
691
692 private void configureResourceManager(BuildRequest request) {
693 ResourceManager resourceMgr = ResourceManager.instance();
694 ExecutionOptions options = request.getOptions(ExecutionOptions.class);
Mark Schallerdffb6ee2015-02-25 20:01:01 +0000695 ResourceSet resources;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100696 if (options.availableResources != null) {
Mark Schallerdffb6ee2015-02-25 20:01:01 +0000697 resources = options.availableResources;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100698 resourceMgr.setRamUtilizationPercentage(100);
699 } else {
Mark Schallerdffb6ee2015-02-25 20:01:01 +0000700 resources = LocalHostCapacity.getLocalHostCapacity();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100701 resourceMgr.setRamUtilizationPercentage(options.ramUtilizationPercentage);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100702 }
Mark Schallerdffb6ee2015-02-25 20:01:01 +0000703
704 resourceMgr.setAvailableResources(ResourceSet.create(
705 resources.getMemoryMb(),
706 resources.getCpuUsage(),
707 resources.getIoUsage(),
708 request.getExecutionOptions().usingLocalTestJobs()
709 ? request.getExecutionOptions().localTestJobs : Integer.MAX_VALUE
710 ));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100711 }
712
713 /**
714 * Writes the cache files to disk, reporting any errors that occurred during
715 * writing.
716 */
717 private void saveCaches(ActionCache actionCache) {
718 long actionCacheSizeInBytes = 0;
719 long actionCacheSaveTime;
720
Nathan Harmatad4803012015-09-08 20:03:22 +0000721 AutoProfiler p = AutoProfiler.profiledAndLogged("Saving action cache", ProfilerTask.INFO, LOG);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100722 try {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100723 actionCacheSizeInBytes = actionCache.save();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100724 } catch (IOException e) {
725 getReporter().handle(Event.error("I/O error while writing action log: " + e.getMessage()));
726 } finally {
Nathan Harmatad4803012015-09-08 20:03:22 +0000727 actionCacheSaveTime = p.completeAndGetElapsedTimeNanos();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100728 }
Ulf Adams633f5392015-09-15 11:13:08 +0000729 env.getEventBus().post(new CachesSavedEvent(
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100730 actionCacheSaveTime, actionCacheSizeInBytes));
731 }
732
733 private ActionInputFileCache createBuildSingleFileCache(Path execRoot) {
734 String cwd = execRoot.getPathString();
735 FileSystem fs = runtime.getDirectories().getFileSystem();
736
737 ActionInputFileCache cache = null;
738 for (BlazeModule module : runtime.getBlazeModules()) {
739 ActionInputFileCache pluggable = module.createActionInputCache(cwd, fs);
740 if (pluggable != null) {
741 Preconditions.checkState(cache == null);
742 cache = pluggable;
743 }
744 }
745
746 if (cache == null) {
747 cache = new SingleBuildFileCache(cwd, fs);
748 }
749 return cache;
750 }
751
752 private Reporter getReporter() {
Ulf Adams633f5392015-09-15 11:13:08 +0000753 return env.getReporter();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100754 }
755
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100756 private Path getWorkspace() {
757 return runtime.getWorkspace();
758 }
759
760 private Path getExecRoot() {
761 return runtime.getExecRoot();
762 }
763}