blob: 997b9e750431d7bcd8df9c6553bed6d33f095e25 [file] [log] [blame]
Damien Martin-Guillerezf88f4d82015-09-25 13:56:55 +00001// Copyright 2014 The Bazel Authors. All rights reserved.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.buildtool;
15
Eric Fellheimere9b41c62016-06-14 15:39:51 +000016import static java.util.concurrent.TimeUnit.MILLISECONDS;
17import static java.util.concurrent.TimeUnit.NANOSECONDS;
18
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010019import com.google.common.base.Joiner;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010020import com.google.common.base.Predicate;
21import com.google.common.base.Stopwatch;
Chloe Calvarin5dc44f02017-03-21 14:40:28 +000022import com.google.common.base.Strings;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010023import com.google.common.collect.HashBasedTable;
Philipp Wollermann45bf15b2015-06-18 13:06:16 +000024import com.google.common.collect.ImmutableList;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010025import com.google.common.collect.ImmutableMap;
Philipp Wollermann1ecdc512016-02-26 21:55:31 +000026import com.google.common.collect.ImmutableMultimap;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010027import com.google.common.collect.ImmutableSet;
28import com.google.common.collect.Iterables;
Philipp Wollermann1ecdc512016-02-26 21:55:31 +000029import com.google.common.collect.Multimap;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010030import com.google.common.collect.Ordering;
31import com.google.common.collect.Table;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010032import com.google.devtools.build.lib.actions.Action;
33import com.google.devtools.build.lib.actions.ActionCacheChecker;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010034import com.google.devtools.build.lib.actions.ActionContextMarker;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010035import com.google.devtools.build.lib.actions.ActionGraph;
36import com.google.devtools.build.lib.actions.ActionInputFileCache;
37import com.google.devtools.build.lib.actions.Artifact;
Ulf Adams3d67e002016-03-29 16:23:01 +000038import com.google.devtools.build.lib.actions.ArtifactFactory;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010039import com.google.devtools.build.lib.actions.BuildFailedException;
40import com.google.devtools.build.lib.actions.ExecException;
41import com.google.devtools.build.lib.actions.ExecutionStrategy;
42import com.google.devtools.build.lib.actions.Executor;
43import com.google.devtools.build.lib.actions.Executor.ActionContext;
44import com.google.devtools.build.lib.actions.ExecutorInitException;
45import com.google.devtools.build.lib.actions.LocalHostCapacity;
46import com.google.devtools.build.lib.actions.ResourceManager;
Mark Schallerdffb6ee2015-02-25 20:01:01 +000047import com.google.devtools.build.lib.actions.ResourceSet;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010048import com.google.devtools.build.lib.actions.SpawnActionContext;
49import com.google.devtools.build.lib.actions.TestExecException;
50import com.google.devtools.build.lib.actions.cache.ActionCache;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010051import com.google.devtools.build.lib.analysis.BuildView.AnalysisResult;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010052import com.google.devtools.build.lib.analysis.ConfiguredTarget;
Googlerbfd4e242016-07-15 22:23:37 +000053import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010054import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010055import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
Lukacs Berki7b2f2e82016-11-23 14:16:43 +000056import com.google.devtools.build.lib.analysis.actions.SymlinkTreeActionContext;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010057import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
58import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
59import com.google.devtools.build.lib.buildtool.buildevent.ExecutionPhaseCompleteEvent;
60import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent;
Kristina Chodorow33aada22016-06-22 14:23:47 +000061import com.google.devtools.build.lib.cmdline.PackageIdentifier;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010062import com.google.devtools.build.lib.events.Event;
63import com.google.devtools.build.lib.events.EventHandler;
64import com.google.devtools.build.lib.events.EventKind;
65import com.google.devtools.build.lib.events.Reporter;
Ulf Adamsdba3c832016-12-21 16:50:02 +000066import com.google.devtools.build.lib.exec.ActionContextConsumer;
67import com.google.devtools.build.lib.exec.ActionContextProvider;
Ulf Adamsccd5c0a2017-01-13 15:46:51 +000068import com.google.devtools.build.lib.exec.ActionInputPrefetcher;
Ulf Adamsdba3c832016-12-21 16:50:02 +000069import com.google.devtools.build.lib.exec.BlazeExecutor;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010070import com.google.devtools.build.lib.exec.CheckUpToDateFilter;
71import com.google.devtools.build.lib.exec.ExecutionOptions;
Ulf Adamsdba3c832016-12-21 16:50:02 +000072import com.google.devtools.build.lib.exec.ExecutorBuilder;
73import com.google.devtools.build.lib.exec.FilesetActionContextImpl;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010074import com.google.devtools.build.lib.exec.OutputService;
75import com.google.devtools.build.lib.exec.SingleBuildFileCache;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010076import com.google.devtools.build.lib.exec.SymlinkTreeStrategy;
Nathan Harmatad4803012015-09-08 20:03:22 +000077import com.google.devtools.build.lib.profiler.AutoProfiler;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010078import com.google.devtools.build.lib.profiler.ProfilePhase;
79import com.google.devtools.build.lib.profiler.Profiler;
80import com.google.devtools.build.lib.profiler.ProfilerTask;
81import com.google.devtools.build.lib.rules.fileset.FilesetActionContext;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010082import com.google.devtools.build.lib.rules.test.TestActionContext;
83import com.google.devtools.build.lib.runtime.BlazeModule;
84import com.google.devtools.build.lib.runtime.BlazeRuntime;
Ulf Adams633f5392015-09-15 11:13:08 +000085import com.google.devtools.build.lib.runtime.CommandEnvironment;
Dmitry Lomove2033b12015-08-19 16:57:49 +000086import com.google.devtools.build.lib.skyframe.AspectValue;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010087import com.google.devtools.build.lib.skyframe.Builder;
88import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010089import com.google.devtools.build.lib.util.AbruptExitException;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010090import com.google.devtools.build.lib.util.ExitCode;
91import com.google.devtools.build.lib.util.LoggingUtil;
Mark Schaller6df81792015-12-10 18:47:47 +000092import com.google.devtools.build.lib.util.Preconditions;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010093import com.google.devtools.build.lib.vfs.FileSystemUtils;
Eric Fellheimerf3b43af2015-11-13 19:51:20 +000094import com.google.devtools.build.lib.vfs.ModifiedFileSet;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010095import com.google.devtools.build.lib.vfs.Path;
96import com.google.devtools.build.lib.vfs.PathFragment;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010097import java.io.IOException;
98import java.io.OutputStream;
99import java.io.PrintWriter;
100import java.util.ArrayList;
101import java.util.Collection;
102import java.util.HashMap;
103import java.util.HashSet;
104import java.util.LinkedHashSet;
105import java.util.List;
106import java.util.Map;
107import java.util.Set;
Philipp Wollermanna068dee2015-06-08 11:15:02 +0000108import java.util.TreeMap;
Eric Fellheimer2db6a742015-04-28 21:38:43 +0000109import java.util.UUID;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100110import java.util.logging.Level;
111import java.util.logging.Logger;
112
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100113/**
114 * This class manages the execution phase. The entry point is {@link #executeBuild}.
115 *
116 * <p>This is only intended for use by {@link BuildTool}.
117 *
118 * <p>This class contains an ActionCache, and refers to the BlazeRuntime's BuildView and
119 * PackageCache.
120 *
121 * @see BuildTool
Kristina Chodorowe423fdb2016-09-15 14:08:08 +0000122 * @see com.google.devtools.build.lib.analysis.BuildView
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100123 */
124public class ExecutionTool {
125 private static class StrategyConverter {
126 private Table<Class<? extends ActionContext>, String, ActionContext> classMap =
127 HashBasedTable.create();
128 private Map<Class<? extends ActionContext>, ActionContext> defaultClassMap =
129 new HashMap<>();
130
131 /**
132 * Aggregates all {@link ActionContext}s that are in {@code contextProviders}.
133 */
134 @SuppressWarnings("unchecked")
135 private StrategyConverter(Iterable<ActionContextProvider> contextProviders) {
136 for (ActionContextProvider provider : contextProviders) {
137 for (ActionContext strategy : provider.getActionContexts()) {
138 ExecutionStrategy annotation =
139 strategy.getClass().getAnnotation(ExecutionStrategy.class);
140 if (annotation != null) {
141 defaultClassMap.put(annotation.contextType(), strategy);
142
143 for (String name : annotation.name()) {
144 classMap.put(annotation.contextType(), name, strategy);
145 }
146 }
147 }
148 }
149 }
150
151 @SuppressWarnings("unchecked")
152 private <T extends ActionContext> T getStrategy(Class<T> clazz, String name) {
153 return (T) (name.isEmpty() ? defaultClassMap.get(clazz) : classMap.get(clazz, name));
154 }
155
156 private String getValidValues(Class<? extends ActionContext> context) {
157 return Joiner.on(", ").join(Ordering.natural().sortedCopy(classMap.row(context).keySet()));
158 }
159
160 private String getUserFriendlyName(Class<? extends ActionContext> context) {
161 ActionContextMarker marker = context.getAnnotation(ActionContextMarker.class);
162 return marker != null
163 ? marker.name()
164 : context.getSimpleName();
165 }
166 }
167
Kristina Chodorowe423fdb2016-09-15 14:08:08 +0000168 static final Logger log = Logger.getLogger(ExecutionTool.class.getName());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100169
Ulf Adams633f5392015-09-15 11:13:08 +0000170 private final CommandEnvironment env;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100171 private final BlazeRuntime runtime;
172 private final BuildRequest request;
173 private BlazeExecutor executor;
Ulf Adams1ac1b992016-11-02 12:50:26 +0000174 private final ActionInputFileCache fileCache;
Ulf Adamsccd5c0a2017-01-13 15:46:51 +0000175 private final ActionInputPrefetcher prefetcher;
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000176 private final ImmutableList<ActionContextProvider> actionContextProviders;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100177
Philipp Wollermanna068dee2015-06-08 11:15:02 +0000178 private Map<String, SpawnActionContext> spawnStrategyMap =
179 new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100180 private List<ActionContext> strategies = new ArrayList<>();
181
Ulf Adams633f5392015-09-15 11:13:08 +0000182 ExecutionTool(CommandEnvironment env, BuildRequest request) throws ExecutorInitException {
183 this.env = env;
184 this.runtime = env.getRuntime();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100185 this.request = request;
186
hlopkoc4134802017-04-04 13:53:14 +0000187
Philipp Wollermann590ea392015-08-25 13:57:33 +0000188 // Create tools before getting the strategies from the modules as some of them need tools to
189 // determine whether the host actually supports certain strategies (e.g. sandboxing).
190 createToolsSymlinks();
191
Ulf Adamsa0e3af42016-10-31 16:52:48 +0000192 ExecutorBuilder builder = new ExecutorBuilder();
193 for (BlazeModule module : runtime.getBlazeModules()) {
194 module.executorInit(env, request, builder);
195 }
196 builder.addActionContextProvider(
197 new FilesetActionContextImpl.Provider(env.getReporter(), env.getWorkspaceName()));
198 builder.addActionContext(new SymlinkTreeStrategy(
199 env.getOutputService(), env.getBlazeWorkspace().getBinTools()));
200 // TODO(philwo) - the ExecutionTool should not add arbitrary dependencies on its own, instead
201 // these dependencies should be added to the ActionContextConsumer of the module that actually
202 // depends on them.
203 builder.addActionContextConsumer(
204 new ActionContextConsumer() {
205 @Override
206 public ImmutableMap<String, String> getSpawnActionContexts() {
207 return ImmutableMap.of();
208 }
209
210 @Override
211 public Multimap<Class<? extends ActionContext>, String> getActionContexts() {
212 return ImmutableMultimap.<Class<? extends ActionContext>, String>builder()
213 .put(FilesetActionContext.class, "")
214 .put(WorkspaceStatusAction.Context.class, "")
215 .put(SymlinkTreeActionContext.class, "")
216 .build();
217 }
218 });
219
Ulf Adams1ac1b992016-11-02 12:50:26 +0000220 ActionInputFileCache cache = builder.getActionInputFileCache();
221 if (cache == null) {
222 // Unfortunately, the exec root cache is not shared with caches in the remote execution
223 // client.
224 cache =
225 new SingleBuildFileCache(
226 env.getExecRoot().getPathString(), env.getDirectories().getFileSystem());
227 }
228 this.fileCache = cache;
Ulf Adamsccd5c0a2017-01-13 15:46:51 +0000229 this.prefetcher = builder.getActionInputPrefetcher();
Ulf Adams1ac1b992016-11-02 12:50:26 +0000230
Ulf Adamsa0e3af42016-10-31 16:52:48 +0000231 this.actionContextProviders = builder.getActionContextProviders();
Ulf Adamsccd5c0a2017-01-13 15:46:51 +0000232 for (ActionContextProvider provider : actionContextProviders) {
233 provider.init(fileCache, prefetcher);
234 }
235
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100236 StrategyConverter strategyConverter = new StrategyConverter(actionContextProviders);
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000237
Ulf Adamsa0e3af42016-10-31 16:52:48 +0000238 for (ActionContextConsumer consumer : builder.getActionContextConsumers()) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100239 // There are many different SpawnActions, and we want to control the action context they use
240 // independently from each other, for example, to run genrules locally and Java compile action
241 // in prod. Thus, for SpawnActions, we decide the action context to use not only based on the
242 // context class, but also the mnemonic of the action.
243 for (Map.Entry<String, String> entry : consumer.getSpawnActionContexts().entrySet()) {
244 SpawnActionContext context =
245 strategyConverter.getStrategy(SpawnActionContext.class, entry.getValue());
246 if (context == null) {
Chloe Calvarin5dc44f02017-03-21 14:40:28 +0000247 String strategy = Strings.emptyToNull(entry.getKey());
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000248 throw makeExceptionForInvalidStrategyValue(
249 entry.getValue(),
Chloe Calvarin5dc44f02017-03-21 14:40:28 +0000250 Joiner.on(' ').skipNulls().join(strategy, "spawn"),
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100251 strategyConverter.getValidValues(SpawnActionContext.class));
252 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100253 spawnStrategyMap.put(entry.getKey(), context);
254 }
255
256 for (Map.Entry<Class<? extends ActionContext>, String> entry :
Philipp Wollermann1ecdc512016-02-26 21:55:31 +0000257 consumer.getActionContexts().entries()) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100258 ActionContext context = strategyConverter.getStrategy(entry.getKey(), entry.getValue());
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000259 if (context == null) {
260 throw makeExceptionForInvalidStrategyValue(
261 entry.getValue(),
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100262 strategyConverter.getUserFriendlyName(entry.getKey()),
263 strategyConverter.getValidValues(entry.getKey()));
264 }
Philipp Wollermann45bf15b2015-06-18 13:06:16 +0000265 strategies.add(context);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100266 }
267 }
268
Luis Fernando Pino Duque9b1fb682016-06-16 10:04:25 +0000269 String testStrategyValue = request.getOptions(ExecutionOptions.class).testStrategy;
270 ActionContext context = strategyConverter.getStrategy(TestActionContext.class,
271 testStrategyValue);
272 if (context == null) {
273 throw makeExceptionForInvalidStrategyValue(testStrategyValue, "test",
274 strategyConverter.getValidValues(TestActionContext.class));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100275 }
Luis Fernando Pino Duque9b1fb682016-06-16 10:04:25 +0000276 strategies.add(context);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100277 }
278
279 private static ExecutorInitException makeExceptionForInvalidStrategyValue(String value,
280 String strategy, String validValues) {
281 return new ExecutorInitException(String.format(
282 "'%s' is an invalid value for %s strategy. Valid values are: %s", value, strategy,
283 validValues), ExitCode.COMMAND_LINE_ERROR);
284 }
285
286 Executor getExecutor() throws ExecutorInitException {
287 if (executor == null) {
288 executor = createExecutor();
289 }
290 return executor;
291 }
292
293 /**
294 * Creates an executor for the current set of blaze runtime, execution options, and request.
295 */
296 private BlazeExecutor createExecutor()
297 throws ExecutorInitException {
298 return new BlazeExecutor(
Ulf Adams94b72db2016-03-30 11:58:37 +0000299 env.getExecRoot(),
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100300 getReporter(),
Ulf Adams633f5392015-09-15 11:13:08 +0000301 env.getEventBus(),
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100302 runtime.getClock(),
303 request,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100304 strategies,
305 spawnStrategyMap,
306 actionContextProviders);
307 }
308
309 void init() throws ExecutorInitException {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100310 getExecutor();
311 }
312
313 void shutdown() {
314 for (ActionContextProvider actionContextProvider : actionContextProviders) {
315 actionContextProvider.executionPhaseEnding();
316 }
317 }
318
319 /**
320 * Performs the execution phase (phase 3) of the build, in which the Builder
321 * is applied to the action graph to bring the targets up to date. (This
322 * function will return prior to execution-proper if --nobuild was specified.)
Eric Fellheimer2db6a742015-04-28 21:38:43 +0000323 * @param buildId UUID of the build id
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100324 * @param analysisResult the analysis phase output
325 * @param buildResult the mutable build result
Lukacs Berki3ea4d442016-01-21 15:15:30 +0000326 * @param packageRoots package roots collected from loading phase and BuildConfigurationCollection
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100327 * creation
328 */
Eric Fellheimer2db6a742015-04-28 21:38:43 +0000329 void executeBuild(UUID buildId, AnalysisResult analysisResult,
Laurent Le Brunf3cf98f2016-06-17 13:36:24 +0000330 BuildResult buildResult,
331 BuildConfigurationCollection configurations,
Googlerbfd4e242016-07-15 22:23:37 +0000332 ImmutableMap<PackageIdentifier, Path> packageRoots,
333 TopLevelArtifactContext topLevelArtifactContext)
Nathan Harmataca06fa22015-07-27 19:54:14 +0000334 throws BuildFailedException, InterruptedException, TestExecException, AbruptExitException {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100335 Stopwatch timer = Stopwatch.createStarted();
hlopkoc4134802017-04-04 13:53:14 +0000336 prepare(packageRoots, analysisResult.getWorkspaceName());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100337
338 ActionGraph actionGraph = analysisResult.getActionGraph();
339
340 // Get top-level artifacts.
341 ImmutableSet<Artifact> additionalArtifacts = analysisResult.getAdditionalArtifactsToBuild();
342
Ulf Adams706b7f22015-10-20 09:06:45 +0000343 OutputService outputService = env.getOutputService();
Eric Fellheimerf3b43af2015-11-13 19:51:20 +0000344 ModifiedFileSet modifiedOutputFiles = ModifiedFileSet.EVERYTHING_MODIFIED;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100345 if (outputService != null) {
Eric Fellheimer1246eb32015-11-18 15:41:38 +0000346 modifiedOutputFiles = outputService.startBuild(buildId,
347 request.getBuildOptions().finalizeActions);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100348 } else {
Kristina Chodorowe423fdb2016-09-15 14:08:08 +0000349 // TODO(bazel-team): this could be just another OutputService
hlopkoc4134802017-04-04 13:53:14 +0000350 startLocalOutputBuild(analysisResult.getWorkspaceName());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100351 }
352
janakr0049e962017-04-05 21:41:23 +0000353 // Must be created after the output path is created above.
354 createActionLogDirectory();
355
Lukacs Berkiad359c62016-01-26 12:59:36 +0000356 List<BuildConfiguration> targetConfigurations = configurations.getTargetConfigurations();
357 BuildConfiguration targetConfiguration = targetConfigurations.size() == 1
358 ? targetConfigurations.get(0) : null;
359 if (targetConfigurations.size() == 1) {
Luis Fernando Pino Duque5a5bd8a2017-02-10 13:50:01 +0000360 String productName = runtime.getProductName();
hlopkoc4134802017-04-04 13:53:14 +0000361 String dirName = env.getWorkspaceName();
362 String workspaceName = analysisResult.getWorkspaceName();
Lukacs Berkiad359c62016-01-26 12:59:36 +0000363 OutputDirectoryLinksUtils.createOutputDirectoryLinks(
hlopkoc4134802017-04-04 13:53:14 +0000364 dirName, env.getWorkspace(), env.getDirectories().getExecRoot(workspaceName),
Luis Fernando Pino Duque5a5bd8a2017-02-10 13:50:01 +0000365 env.getDirectories().getOutputPath(workspaceName), getReporter(), targetConfiguration,
366 request.getBuildOptions().getSymlinkPrefix(productName), productName);
Lukacs Berki85c63c42016-01-22 09:25:20 +0000367 }
368
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100369 ActionCache actionCache = getActionCache();
Ulf Adams80613022015-09-16 09:11:33 +0000370 SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
Kristina Chodorowe423fdb2016-09-15 14:08:08 +0000371 Builder builder = createBuilder(
Ulf Adams1ac1b992016-11-02 12:50:26 +0000372 request, actionCache, skyframeExecutor, modifiedOutputFiles);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100373
374 //
375 // Execution proper. All statements below are logically nested in
376 // begin/end pairs. No early returns or exceptions please!
377 //
378
379 Collection<ConfiguredTarget> configuredTargets = buildResult.getActualTargets();
Ulf Adams633f5392015-09-15 11:13:08 +0000380 env.getEventBus().post(new ExecutionStartingEvent(configuredTargets));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100381
382 getReporter().handle(Event.progress("Building..."));
383
384 // Conditionally record dependency-checker log:
385 ExplanationHandler explanationHandler =
386 installExplanationHandler(request.getBuildOptions().explanationPath,
387 request.getOptionsDescription());
388
389 Set<ConfiguredTarget> builtTargets = new HashSet<>();
Janak Ramakrishnana5d8d092015-12-08 18:32:47 +0000390 Collection<AspectValue> aspects = analysisResult.getAspects();
391
392 Iterable<Artifact> allArtifactsForProviders =
393 Iterables.concat(
394 additionalArtifacts,
395 TopLevelArtifactHelper.getAllArtifactsToBuild(
396 analysisResult.getTargetsToBuild(), analysisResult.getTopLevelContext())
397 .getAllArtifacts(),
398 TopLevelArtifactHelper.getAllArtifactsToBuildFromAspects(
399 aspects, analysisResult.getTopLevelContext())
400 .getAllArtifacts(),
401 //TODO(dslomov): Artifacts to test from aspects?
402 TopLevelArtifactHelper.getAllArtifactsToTest(analysisResult.getTargetsToTest()));
403
404 if (request.isRunningInEmacs()) {
405 // The syntax of this message is tightly constrained by lisp/progmodes/compile.el in emacs
406 request.getOutErr().printErrLn("blaze: Entering directory `" + getExecRoot() + "/'");
407 }
408 boolean buildCompleted = false;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100409 try {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100410 for (ActionContextProvider actionContextProvider : actionContextProviders) {
Ulf Adamsccd5c0a2017-01-13 15:46:51 +0000411 actionContextProvider.executionPhaseStarting(actionGraph, allArtifactsForProviders);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100412 }
413 executor.executionPhaseStarting();
414 skyframeExecutor.drainChangedFiles();
415
416 if (request.getViewOptions().discardAnalysisCache) {
417 // Free memory by removing cache entries that aren't going to be needed. Note that in
418 // skyframe full, this destroys the action graph as well, so we can only do it after the
419 // action graph is no longer needed.
janakrdb4dec22017-03-28 22:39:35 +0000420 env.getSkyframeBuildView()
421 .clearAnalysisCache(analysisResult.getTargetsToBuild(), analysisResult.getAspects());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100422 }
423
424 configureResourceManager(request);
425
426 Profiler.instance().markPhase(ProfilePhase.EXECUTE);
427
Dmitry Lomove2033b12015-08-19 16:57:49 +0000428 builder.buildArtifacts(
Ulf Adams5b9009b2015-09-24 09:52:53 +0000429 env.getReporter(),
Dmitry Lomove2033b12015-08-19 16:57:49 +0000430 additionalArtifacts,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100431 analysisResult.getParallelTests(),
432 analysisResult.getExclusiveTests(),
433 analysisResult.getTargetsToBuild(),
Dmitry Lomove2033b12015-08-19 16:57:49 +0000434 analysisResult.getAspects(),
435 executor,
436 builtTargets,
Miguel Alcon Pinto7cf23652015-03-10 21:27:48 +0000437 request.getBuildOptions().explanationPath != null,
Googlerbfd4e242016-07-15 22:23:37 +0000438 env.getBlazeWorkspace().getLastExecutionTimeRange(),
439 topLevelArtifactContext);
Janak Ramakrishnana5d8d092015-12-08 18:32:47 +0000440 buildCompleted = true;
441 } catch (BuildFailedException | TestExecException e) {
442 buildCompleted = true;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100443 throw e;
444 } finally {
Ulf Adamsb5146102015-10-20 08:57:26 +0000445 env.recordLastExecutionTime();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100446 if (request.isRunningInEmacs()) {
447 request.getOutErr().printErrLn("blaze: Leaving directory `" + getExecRoot() + "/'");
448 }
Janak Ramakrishnana5d8d092015-12-08 18:32:47 +0000449 if (buildCompleted) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100450 getReporter().handle(Event.progress("Building complete."));
451 }
452
Ulf Adams633f5392015-09-15 11:13:08 +0000453 env.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap.<String, Long> of(), 0L,
Miguel Alcon Pinto9186f682015-04-09 21:49:34 +0000454 skyframeExecutor.getOutputDirtyFilesAndClear(),
455 skyframeExecutor.getModifiedFilesDuringPreviousBuildAndClear()));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100456
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100457 executor.executionPhaseEnding();
458 for (ActionContextProvider actionContextProvider : actionContextProviders) {
459 actionContextProvider.executionPhaseEnding();
460 }
461
462 Profiler.instance().markPhase(ProfilePhase.FINISH);
463
Janak Ramakrishnana5d8d092015-12-08 18:32:47 +0000464 if (buildCompleted) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100465 saveCaches(actionCache);
466 }
467
Nathan Harmatad4803012015-09-08 20:03:22 +0000468 try (AutoProfiler p = AutoProfiler.profiled("Show results", ProfilerTask.INFO)) {
Ulf Adams7a3fe522015-12-04 10:48:56 +0000469 buildResult.setSuccessfulTargets(
470 determineSuccessfulTargets(configuredTargets, builtTargets, timer));
Googler3ca7b782015-10-12 15:44:09 +0000471 BuildResultPrinter buildResultPrinter = new BuildResultPrinter(env);
472 buildResultPrinter.showBuildResult(
473 request, buildResult, configuredTargets, analysisResult.getAspects());
Nathan Harmatad4803012015-09-08 20:03:22 +0000474 }
Googler3ca7b782015-10-12 15:44:09 +0000475
476 try (AutoProfiler p = AutoProfiler.profiled("Show artifacts", ProfilerTask.INFO)) {
477 if (request.getBuildOptions().showArtifacts) {
478 BuildResultPrinter buildResultPrinter = new BuildResultPrinter(env);
479 buildResultPrinter.showArtifacts(
480 request, configuredTargets, analysisResult.getAspects());
481 }
482 }
483
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100484 if (explanationHandler != null) {
485 uninstallExplanationHandler(explanationHandler);
486 }
487 // Finalize output service last, so that if we do throw an exception, we know all the other
488 // code has already run.
Ulf Adams706b7f22015-10-20 09:06:45 +0000489 if (env.getOutputService() != null) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100490 boolean isBuildSuccessful =
491 buildResult.getSuccessfulTargets().size() == configuredTargets.size();
Ulf Adams706b7f22015-10-20 09:06:45 +0000492 env.getOutputService().finalizeBuild(isBuildSuccessful);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100493 }
494 }
495 }
496
hlopkoc4134802017-04-04 13:53:14 +0000497 private void prepare(ImmutableMap<PackageIdentifier, Path> packageRoots, String workspaceName)
Laurent Le Brunf3cf98f2016-06-17 13:36:24 +0000498 throws ExecutorInitException {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100499 // Prepare for build.
500 Profiler.instance().markPhase(ProfilePhase.PREPARE);
501
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100502 // Plant the symlink forest.
Kristina Chodorow52bc57c2016-06-20 14:27:56 +0000503 try {
Dmitry Lomove36a66c2017-02-17 14:48:48 +0000504 new SymlinkForest(
hlopkoc4134802017-04-04 13:53:14 +0000505 packageRoots, getExecRoot(), runtime.getProductName(), workspaceName)
Kristina Chodorow36a6c172016-07-22 15:18:07 +0000506 .plantSymlinkForest();
Kristina Chodorow52bc57c2016-06-20 14:27:56 +0000507 } catch (IOException e) {
508 throw new ExecutorInitException("Source forest creation failed", e);
509 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100510 }
511
512 private void createToolsSymlinks() throws ExecutorInitException {
513 try {
hlopkoc4134802017-04-04 13:53:14 +0000514 env.getBlazeWorkspace().getBinTools().setupBuildTools();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100515 } catch (ExecException e) {
Nathan Harmataca06fa22015-07-27 19:54:14 +0000516 throw new ExecutorInitException("Tools symlink creation failed", e);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100517 }
518 }
519
Nathan Harmataca06fa22015-07-27 19:54:14 +0000520 private void createActionLogDirectory() throws ExecutorInitException {
janakr0049e962017-04-05 21:41:23 +0000521 Path directory = env.getActionConsoleOutputDirectory();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100522 try {
523 if (directory.exists()) {
524 FileSystemUtils.deleteTree(directory);
525 }
janakr0049e962017-04-05 21:41:23 +0000526 FileSystemUtils.createDirectoryAndParents(directory);
Nathan Harmataca06fa22015-07-27 19:54:14 +0000527 } catch (IOException e) {
528 throw new ExecutorInitException("Couldn't delete action output directory", e);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100529 }
530 }
531
532 /**
533 * Prepare for a local output build.
534 */
hlopkoc4134802017-04-04 13:53:14 +0000535 private void startLocalOutputBuild(String workspaceName) throws ExecutorInitException {
Nathan Harmatad4803012015-09-08 20:03:22 +0000536 try (AutoProfiler p = AutoProfiler.profiled("Starting local output build", ProfilerTask.INFO)) {
hlopkoc4134802017-04-04 13:53:14 +0000537 Path outputPath = env.getDirectories().getOutputPath(workspaceName);
Ulf Adams94b72db2016-03-30 11:58:37 +0000538 Path localOutputPath = env.getDirectories().getLocalOutputPath();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100539
540 if (outputPath.isSymbolicLink()) {
Nathan Harmataca06fa22015-07-27 19:54:14 +0000541 try {
542 // Remove the existing symlink first.
543 outputPath.delete();
544 if (localOutputPath.exists()) {
545 // Pre-existing local output directory. Move to outputPath.
546 localOutputPath.renameTo(outputPath);
547 }
548 } catch (IOException e) {
549 throw new ExecutorInitException("Couldn't handle local output directory symlinks", e);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100550 }
551 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100552 }
553 }
554
555 /**
556 * If a path is supplied, creates and installs an ExplanationHandler. Returns
557 * an instance on success. Reports an error and returns null otherwise.
558 */
559 private ExplanationHandler installExplanationHandler(PathFragment explanationPath,
560 String allOptions) {
561 if (explanationPath == null) {
562 return null;
563 }
564 ExplanationHandler handler;
565 try {
566 handler = new ExplanationHandler(
567 getWorkspace().getRelative(explanationPath).getOutputStream(),
568 allOptions);
569 } catch (IOException e) {
570 getReporter().handle(Event.warn(String.format(
571 "Cannot write explanation of rebuilds to file '%s': %s",
572 explanationPath, e.getMessage())));
573 return null;
574 }
575 getReporter().handle(
576 Event.info("Writing explanation of rebuilds to '" + explanationPath + "'"));
577 getReporter().addHandler(handler);
578 return handler;
579 }
580
581 /**
582 * Uninstalls the specified ExplanationHandler (if any) and closes the log
583 * file.
584 */
585 private void uninstallExplanationHandler(ExplanationHandler handler) {
586 if (handler != null) {
587 getReporter().removeHandler(handler);
588 handler.log.close();
589 }
590 }
591
592 /**
593 * An ErrorEventListener implementation that records DEPCHECKER events into a log
594 * file, iff the --explain flag is specified during a build.
595 */
596 private static class ExplanationHandler implements EventHandler {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100597 private final PrintWriter log;
598
599 private ExplanationHandler(OutputStream log, String optionsDescription) {
600 this.log = new PrintWriter(log);
601 this.log.println("Build options: " + optionsDescription);
602 }
603
604
605 @Override
606 public void handle(Event event) {
607 if (event.getKind() == EventKind.DEPCHECKER) {
608 log.println(event.getMessage());
609 }
610 }
611 }
612
613 /**
614 * Computes the result of the build. Sets the list of successful (up-to-date)
615 * targets in the request object.
616 *
617 * @param configuredTargets The configured targets whose artifacts are to be
618 * built.
619 * @param timer A timer that was started when the execution phase started.
620 */
Ulf Adams7a3fe522015-12-04 10:48:56 +0000621 private Collection<ConfiguredTarget> determineSuccessfulTargets(
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100622 Collection<ConfiguredTarget> configuredTargets, Set<ConfiguredTarget> builtTargets,
623 Stopwatch timer) {
624 // Maintain the ordering by copying builtTargets into a LinkedHashSet in the same iteration
625 // order as configuredTargets.
626 Collection<ConfiguredTarget> successfulTargets = new LinkedHashSet<>();
627 for (ConfiguredTarget target : configuredTargets) {
628 if (builtTargets.contains(target)) {
629 successfulTargets.add(target);
630 }
631 }
Ulf Adams633f5392015-09-15 11:13:08 +0000632 env.getEventBus().post(
Eric Fellheimere9b41c62016-06-14 15:39:51 +0000633 new ExecutionPhaseCompleteEvent(timer.stop().elapsed(MILLISECONDS)));
Ulf Adams7a3fe522015-12-04 10:48:56 +0000634 return successfulTargets;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100635 }
636
Dmitry Lomov56c7d552015-09-07 14:57:47 +0000637
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100638 private ActionCache getActionCache() throws LocalEnvironmentException {
639 try {
Ulf Adams80613022015-09-16 09:11:33 +0000640 return env.getPersistentActionCache();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100641 } catch (IOException e) {
642 // TODO(bazel-team): (2010) Ideally we should just remove all cache data and reinitialize
643 // caches.
644 LoggingUtil.logToRemote(Level.WARNING, "Failed to initialize action cache: "
645 + e.getMessage(), e);
646 throw new LocalEnvironmentException("couldn't create action cache: " + e.getMessage()
647 + ". If error persists, use 'blaze clean'");
648 }
649 }
650
651 private Builder createBuilder(BuildRequest request,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100652 ActionCache actionCache,
Eric Fellheimerf3b43af2015-11-13 19:51:20 +0000653 SkyframeExecutor skyframeExecutor,
Ulf Adams1ac1b992016-11-02 12:50:26 +0000654 ModifiedFileSet modifiedOutputFiles) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100655 BuildRequest.BuildRequestOptions options = request.getBuildOptions();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100656 boolean keepGoing = request.getViewOptions().keepGoing;
657
janakr0049e962017-04-05 21:41:23 +0000658 Path actionOutputRoot = env.getActionConsoleOutputDirectory();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100659 Predicate<Action> executionFilter = CheckUpToDateFilter.fromOptions(
660 request.getOptions(ExecutionOptions.class));
661
662 // jobs should have been verified in BuildRequest#validateOptions().
663 Preconditions.checkState(options.jobs >= -1);
664 int actualJobs = options.jobs == 0 ? 1 : options.jobs; // Treat 0 jobs as a single task.
665
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100666 skyframeExecutor.setActionOutputRoot(actionOutputRoot);
Ulf Adams3d67e002016-03-29 16:23:01 +0000667 ArtifactFactory artifactFactory = env.getSkyframeBuildView().getArtifactFactory();
Googler2f111922017-02-28 20:58:45 +0000668 return new SkyframeBuilder(
669 skyframeExecutor,
670 new ActionCacheChecker(
671 actionCache,
672 artifactFactory,
673 executionFilter,
674 ActionCacheChecker.CacheConfig.builder()
675 .setEnabled(options.useActionCache)
676 .setVerboseExplanations(options.verboseExplanations)
677 .build()),
678 keepGoing,
679 actualJobs,
Nathan Harmata60108832016-03-25 08:02:42 +0000680 request.getPackageCacheOptions().checkOutputFiles
Googler2f111922017-02-28 20:58:45 +0000681 ? modifiedOutputFiles
682 : ModifiedFileSet.NOTHING_MODIFIED,
683 options.finalizeActions,
684 fileCache,
685 request.getBuildOptions().progressReportInterval);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100686 }
687
688 private void configureResourceManager(BuildRequest request) {
689 ResourceManager resourceMgr = ResourceManager.instance();
690 ExecutionOptions options = request.getOptions(ExecutionOptions.class);
Mark Schallerdffb6ee2015-02-25 20:01:01 +0000691 ResourceSet resources;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100692 if (options.availableResources != null) {
Mark Schallerdffb6ee2015-02-25 20:01:01 +0000693 resources = options.availableResources;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100694 resourceMgr.setRamUtilizationPercentage(100);
695 } else {
Mark Schallerdffb6ee2015-02-25 20:01:01 +0000696 resources = LocalHostCapacity.getLocalHostCapacity();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100697 resourceMgr.setRamUtilizationPercentage(options.ramUtilizationPercentage);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100698 }
Mark Schallerdffb6ee2015-02-25 20:01:01 +0000699
700 resourceMgr.setAvailableResources(ResourceSet.create(
701 resources.getMemoryMb(),
702 resources.getCpuUsage(),
703 resources.getIoUsage(),
704 request.getExecutionOptions().usingLocalTestJobs()
705 ? request.getExecutionOptions().localTestJobs : Integer.MAX_VALUE
706 ));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100707 }
708
709 /**
710 * Writes the cache files to disk, reporting any errors that occurred during
711 * writing.
712 */
713 private void saveCaches(ActionCache actionCache) {
714 long actionCacheSizeInBytes = 0;
Eric Fellheimere9b41c62016-06-14 15:39:51 +0000715 long actionCacheSaveTimeInMs;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100716
Kristina Chodorowe423fdb2016-09-15 14:08:08 +0000717 AutoProfiler p = AutoProfiler.profiledAndLogged("Saving action cache", ProfilerTask.INFO, log);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100718 try {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100719 actionCacheSizeInBytes = actionCache.save();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100720 } catch (IOException e) {
721 getReporter().handle(Event.error("I/O error while writing action log: " + e.getMessage()));
722 } finally {
Eric Fellheimere9b41c62016-06-14 15:39:51 +0000723 actionCacheSaveTimeInMs =
724 MILLISECONDS.convert(p.completeAndGetElapsedTimeNanos(), NANOSECONDS);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100725 }
Ulf Adams633f5392015-09-15 11:13:08 +0000726 env.getEventBus().post(new CachesSavedEvent(
Eric Fellheimere9b41c62016-06-14 15:39:51 +0000727 actionCacheSaveTimeInMs, actionCacheSizeInBytes));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100728 }
729
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100730 private Reporter getReporter() {
Ulf Adams633f5392015-09-15 11:13:08 +0000731 return env.getReporter();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100732 }
733
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100734 private Path getWorkspace() {
Ulf Adams94b72db2016-03-30 11:58:37 +0000735 return env.getWorkspace();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100736 }
737
738 private Path getExecRoot() {
Ulf Adams94b72db2016-03-30 11:58:37 +0000739 return env.getExecRoot();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100740 }
741}