blob: ec049cd5fdd2d3cf62e2a1a45ea2f00e75954be0 [file] [log] [blame]
Damien Martin-Guillerezf88f4d82015-09-25 13:56:55 +00001// Copyright 2014 The Bazel Authors. All rights reserved.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +01002//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package com.google.devtools.build.lib.analysis;
16
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010017import com.google.common.annotations.VisibleForTesting;
18import com.google.common.base.Function;
tomlua155b532017-11-08 20:12:47 +010019import com.google.common.base.Preconditions;
Greg Estren7883bf12016-11-18 20:03:43 +000020import com.google.common.collect.ArrayListMultimap;
janakr75bc18a2018-07-13 00:52:17 -070021import com.google.common.collect.HashMultimap;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010022import com.google.common.collect.ImmutableList;
23import com.google.common.collect.ImmutableMap;
24import com.google.common.collect.ImmutableSet;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010025import com.google.common.collect.Lists;
Greg Estren7883bf12016-11-18 20:03:43 +000026import com.google.common.collect.Multimap;
janakr75bc18a2018-07-13 00:52:17 -070027import com.google.common.collect.SetMultimap;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010028import com.google.common.collect.Sets;
29import com.google.common.eventbus.EventBus;
Rumou Duan33bab462016-04-25 17:55:12 +000030import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010031import com.google.devtools.build.lib.actions.ActionGraph;
janakr93e3eea2017-03-30 22:09:37 +000032import com.google.devtools.build.lib.actions.ActionLookupValue;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010033import com.google.devtools.build.lib.actions.Artifact;
34import com.google.devtools.build.lib.actions.ArtifactFactory;
Eric Fellheimer9c1e12f2015-08-06 20:23:55 +000035import com.google.devtools.build.lib.actions.ArtifactOwner;
janakrae323982017-09-29 21:11:53 +020036import com.google.devtools.build.lib.actions.PackageRoots;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010037import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
38import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
gregce45ae0962017-07-22 00:11:13 +020039import com.google.devtools.build.lib.analysis.constraints.TopLevelConstraintSemantics;
ulfjackab21d182017-08-10 15:36:14 +020040import com.google.devtools.build.lib.analysis.test.CoverageReportActionFactory;
41import com.google.devtools.build.lib.analysis.test.CoverageReportActionFactory.CoverageReportActionsWrapper;
42import com.google.devtools.build.lib.analysis.test.InstrumentedFilesProvider;
Lukacs Berki6e91eb92015-09-21 09:12:37 +000043import com.google.devtools.build.lib.cmdline.Label;
Lukacs Berki6916be22015-02-19 13:36:06 +000044import com.google.devtools.build.lib.collect.nestedset.NestedSet;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010045import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010046import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010047import com.google.devtools.build.lib.events.Event;
Klaus Aehlig777b30d2017-02-24 16:30:15 +000048import com.google.devtools.build.lib.events.ExtendedEventHandler;
Carmi Grushkodf9e5e12016-11-08 23:07:57 +000049import com.google.devtools.build.lib.packages.AspectClass;
Dmitry Lomov15756522016-12-16 16:52:37 +000050import com.google.devtools.build.lib.packages.AspectDescriptor;
Dmitry Lomovca9bfa42016-11-15 13:22:36 +000051import com.google.devtools.build.lib.packages.AspectParameters;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010052import com.google.devtools.build.lib.packages.Attribute;
Dmitry Lomov8ff06452015-10-21 19:16:30 +000053import com.google.devtools.build.lib.packages.NativeAspectClass;
Cal Peyser8a638d52017-03-24 15:44:24 +000054import com.google.devtools.build.lib.packages.NoSuchPackageException;
55import com.google.devtools.build.lib.packages.NoSuchTargetException;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010056import com.google.devtools.build.lib.packages.Rule;
57import com.google.devtools.build.lib.packages.Target;
58import com.google.devtools.build.lib.packages.TargetUtils;
nharmatac4335cf2017-07-18 21:39:13 +020059import com.google.devtools.build.lib.pkgcache.PackageManager.PackageManagerStatistics;
Dmitry Lomove2033b12015-08-19 16:57:49 +000060import com.google.devtools.build.lib.skyframe.AspectValue;
Carmi Grushkodf9e5e12016-11-08 23:07:57 +000061import com.google.devtools.build.lib.skyframe.AspectValue.AspectKey;
Dmitry Lomovc15ba2e2015-10-30 15:50:01 +000062import com.google.devtools.build.lib.skyframe.AspectValue.AspectValueKey;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010063import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
64import com.google.devtools.build.lib.skyframe.CoverageReportValue;
Dmitry Lomove2033b12015-08-19 16:57:49 +000065import com.google.devtools.build.lib.skyframe.SkyframeAnalysisResult;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010066import com.google.devtools.build.lib.skyframe.SkyframeBuildView;
67import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
ulfjack12bb59f52018-07-25 07:19:24 -070068import com.google.devtools.build.lib.skyframe.TargetPatternPhaseValue;
Dmitry Lomov51aafc12016-11-18 14:02:54 +000069import com.google.devtools.build.lib.syntax.SkylarkImport;
70import com.google.devtools.build.lib.syntax.SkylarkImports;
71import com.google.devtools.build.lib.syntax.SkylarkImports.SkylarkImportSyntaxException;
Klaus Aehligc27b4da2017-09-01 10:33:22 +020072import com.google.devtools.build.lib.util.Pair;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010073import com.google.devtools.build.lib.util.RegexFilter;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010074import com.google.devtools.build.skyframe.SkyKey;
Eric Fellheimer9c1e12f2015-08-06 20:23:55 +000075import com.google.devtools.build.skyframe.WalkableGraph;
Dmitry Lomove2033b12015-08-19 16:57:49 +000076import java.util.ArrayList;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010077import java.util.Collection;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010078import java.util.HashSet;
79import java.util.LinkedHashSet;
80import java.util.List;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010081import java.util.Set;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010082import java.util.logging.Logger;
mjhalupka7675dbd2018-01-29 12:46:58 -080083import java.util.stream.Collectors;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010084import javax.annotation.Nullable;
85
86/**
ulfjack865b6212018-06-14 03:41:55 -070087 * The BuildView presents a semantically-consistent and transitively-closed
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +010088 * dependency graph for some set of packages.
89 *
90 * <h2>Package design</h2>
91 *
92 * <p>This package contains the Blaze dependency analysis framework (aka
93 * "analysis phase"). The goal of this code is to perform semantic analysis of
94 * all of the build targets required for a given build, to report
95 * errors/warnings for any problems in the input, and to construct an "action
96 * graph" (see {@code lib.actions} package) correctly representing the work to
97 * be done during the execution phase of the build.
98 *
99 * <p><b>Configurations</b> the inputs to a build come from two sources: the
100 * intrinsic inputs, specified in the BUILD file, are called <em>targets</em>.
101 * The environmental inputs, coming from the build tool, the command-line, or
102 * configuration files, are called the <em>configuration</em>. Only when a
103 * target and a configuration are combined is there sufficient information to
104 * perform a build. </p>
105 *
106 * <p>Targets are implemented by the {@link Target} hierarchy in the {@code
107 * lib.packages} code. Configurations are implemented by {@link
108 * BuildConfiguration}. The pair of these together is represented by an
109 * instance of class {@link ConfiguredTarget}; this is the root of a hierarchy
110 * with different implementations for each kind of target: source file, derived
111 * file, rules, etc.
112 *
113 * <p>The framework code in this package (as opposed to its subpackages) is
114 * responsible for constructing the {@code ConfiguredTarget} graph for a given
115 * target and configuration, taking care of such issues as:
116 * <ul>
117 * <li>caching common subgraphs.
118 * <li>detecting and reporting cycles.
119 * <li>correct propagation of errors through the graph.
120 * <li>reporting universal errors, such as dependencies from production code
121 * to tests, or to experimental branches.
122 * <li>capturing and replaying errors.
123 * <li>maintaining the graph from one build to the next to
124 * avoid unnecessary recomputation.
125 * <li>checking software licenses.
126 * </ul>
127 *
128 * <p>See also {@link ConfiguredTarget} which documents some important
129 * invariants.
130 */
131public class BuildView {
lberki998ec922017-09-05 10:08:25 +0200132 private static final Logger logger = Logger.getLogger(BuildView.class.getName());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100133
134 private final BlazeDirectories directories;
135
136 private final SkyframeExecutor skyframeExecutor;
137 private final SkyframeBuildView skyframeBuildView;
138
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100139 private final ConfiguredRuleClassProvider ruleClassProvider;
140
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100141 /**
142 * A factory class to create the coverage report action. May be null.
143 */
144 @Nullable private final CoverageReportActionFactory coverageReportActionFactory;
145
Ulf Adams59dbf682015-09-17 11:36:43 +0000146 public BuildView(BlazeDirectories directories,
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100147 ConfiguredRuleClassProvider ruleClassProvider,
148 SkyframeExecutor skyframeExecutor,
Ulf Adams5a96d532015-09-26 09:00:57 +0000149 CoverageReportActionFactory coverageReportActionFactory) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100150 this.directories = directories;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100151 this.coverageReportActionFactory = coverageReportActionFactory;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100152 this.ruleClassProvider = ruleClassProvider;
153 this.skyframeExecutor = Preconditions.checkNotNull(skyframeExecutor);
Ulf Adams89eefd72015-09-23 08:00:43 +0000154 this.skyframeBuildView = skyframeExecutor.getSkyframeBuildView();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100155 }
156
ulfjack865b6212018-06-14 03:41:55 -0700157 /** The number of targets freshly evaluated in the last analysis run. */
158 public int getTargetsVisited() {
159 return skyframeBuildView.getEvaluatedTargetKeys().size();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100160 }
161
tomlua0e8aae2018-06-19 15:07:26 -0700162 public int getActionsConstructed() {
163 return skyframeBuildView.getEvaluatedActionCount();
164 }
165
ulfjack865b6212018-06-14 03:41:55 -0700166 public PackageManagerStatistics getAndClearPkgManagerStatistics() {
167 return skyframeExecutor.getPackageManager().getAndClearStatistics();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100168 }
169
ulfjack865b6212018-06-14 03:41:55 -0700170 private ArtifactFactory getArtifactFactory() {
Ulf Adams89eefd72015-09-23 08:00:43 +0000171 return skyframeBuildView.getArtifactFactory();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100172 }
173
mjhalupka7675dbd2018-01-29 12:46:58 -0800174 /** Returns the collection of configured targets corresponding to any of the provided targets. */
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100175 @VisibleForTesting
mjhalupkae253f832018-06-13 14:19:26 -0700176 static LinkedHashSet<ConfiguredTarget> filterTestsByTargets(
ulfjack12bb59f52018-07-25 07:19:24 -0700177 Collection<ConfiguredTarget> targets, Set<Label> allowedTargetLabels) {
mjhalupka7675dbd2018-01-29 12:46:58 -0800178 return targets
179 .stream()
mjhalupkae253f832018-06-13 14:19:26 -0700180 .filter(ct -> allowedTargetLabels.contains(ct.getLabel()))
181 .collect(Collectors.toCollection(LinkedHashSet::new));
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100182 }
183
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100184 @ThreadCompatible
Dmitry Lomove2033b12015-08-19 16:57:49 +0000185 public AnalysisResult update(
ulfjack12bb59f52018-07-25 07:19:24 -0700186 TargetPatternPhaseValue loadingResult,
Dmitry Lomove2033b12015-08-19 16:57:49 +0000187 BuildConfigurationCollection configurations,
188 List<String> aspects,
ulfjackb5daae32018-06-13 07:59:33 -0700189 AnalysisOptions viewOptions,
juliexxia8a2baea2017-12-18 14:33:45 -0800190 boolean keepGoing,
juliexxiaf8b57c52017-12-21 11:37:46 -0800191 int loadingPhaseThreads,
Dmitry Lomove2033b12015-08-19 16:57:49 +0000192 TopLevelArtifactContext topLevelOptions,
Klaus Aehlig777b30d2017-02-24 16:30:15 +0000193 ExtendedEventHandler eventHandler,
Ulf Adams8c4ae672016-03-30 11:20:41 +0000194 EventBus eventBus)
Dmitry Lomove2033b12015-08-19 16:57:49 +0000195 throws ViewCreationFailedException, InterruptedException {
lberki998ec922017-09-05 10:08:25 +0200196 logger.info("Starting analysis");
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100197 pollInterruptedStatus();
198
199 skyframeBuildView.resetEvaluatedConfiguredTargetKeysSet();
tomlua0e8aae2018-06-19 15:07:26 -0700200 skyframeBuildView.resetEvaluationActionCount();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100201
ulfjack12bb59f52018-07-25 07:19:24 -0700202 // TODO(ulfjack): Expensive. Maybe we don't actually need the targets, only the labels?
203 Collection<Target> targets =
204 loadingResult.getTargets(eventHandler, skyframeExecutor.getPackageManager());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100205 eventBus.post(new AnalysisPhaseStartedEvent(targets));
206
tomlu7f220662018-04-27 16:07:24 -0700207 skyframeBuildView.setConfigurations(eventHandler, configurations);
Marian Lobur86bd4fd2015-09-16 10:01:38 +0000208
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100209 // Determine the configurations.
Greg Estren7971e672016-06-01 23:22:48 +0000210 List<TargetAndConfiguration> topLevelTargetsWithConfigs =
juliexxia8ee423d2017-10-18 16:36:57 -0400211 AnalysisUtils.getTargetsWithConfigs(
212 configurations, targets, eventHandler, ruleClassProvider, skyframeExecutor);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100213
Klaus Aehlig71c993b2017-05-09 07:49:46 -0400214 // Report the generated association of targets to configurations
215 Multimap<Label, BuildConfiguration> byLabel =
216 ArrayListMultimap.<Label, BuildConfiguration>create();
217 for (TargetAndConfiguration pair : topLevelTargetsWithConfigs) {
218 byLabel.put(pair.getLabel(), pair.getConfiguration());
219 }
Klaus Aehlig2a92c902017-08-11 14:03:31 +0200220 for (Target target : targets) {
221 eventBus.post(new TargetConfiguredEvent(target, byLabel.get(target.getLabel())));
Klaus Aehlig71c993b2017-05-09 07:49:46 -0400222 }
223
janakr573807d2018-01-11 14:02:35 -0800224 List<ConfiguredTargetKey> topLevelCtKeys =
225 Lists.transform(
226 topLevelTargetsWithConfigs,
227 new Function<TargetAndConfiguration, ConfiguredTargetKey>() {
228 @Override
229 public ConfiguredTargetKey apply(TargetAndConfiguration node) {
230 return ConfiguredTargetKey.of(node.getLabel(), node.getConfiguration());
231 }
232 });
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100233
Klaus Aehligc27b4da2017-09-01 10:33:22 +0200234 Multimap<Pair<Label, String>, BuildConfiguration> aspectConfigurations =
235 ArrayListMultimap.create();
Klaus Aehlig71c993b2017-05-09 07:49:46 -0400236
Dmitry Lomovc15ba2e2015-10-30 15:50:01 +0000237 List<AspectValueKey> aspectKeys = new ArrayList<>();
Dmitry Lomove2033b12015-08-19 16:57:49 +0000238 for (String aspect : aspects) {
Dmitry Lomov0b832ce2015-10-20 10:03:14 +0000239 // Syntax: label%aspect
240 int delimiterPosition = aspect.indexOf('%');
241 if (delimiterPosition >= 0) {
John Fielda97e17f2015-11-13 02:19:52 +0000242 // TODO(jfield): For consistency with Skylark loads, the aspect should be specified
laurentlb940dbc52018-03-01 08:07:14 -0800243 // as an absolute label.
244 // We convert it for compatibility reasons (this will be removed in the future).
Dmitry Lomov51aafc12016-11-18 14:02:54 +0000245 String bzlFileLoadLikeString = aspect.substring(0, delimiterPosition);
246 if (!bzlFileLoadLikeString.startsWith("//") && !bzlFileLoadLikeString.startsWith("@")) {
247 // "Legacy" behavior of '--aspects' parameter.
laurentlb940dbc52018-03-01 08:07:14 -0800248 if (bzlFileLoadLikeString.startsWith("/")) {
249 bzlFileLoadLikeString = bzlFileLoadLikeString.substring(1);
250 }
251 int lastSlashPosition = bzlFileLoadLikeString.lastIndexOf('/');
252 if (lastSlashPosition >= 0) {
253 bzlFileLoadLikeString =
254 "//"
255 + bzlFileLoadLikeString.substring(0, lastSlashPosition)
256 + ":"
257 + bzlFileLoadLikeString.substring(lastSlashPosition + 1);
258 } else {
259 bzlFileLoadLikeString = "//:" + bzlFileLoadLikeString;
260 }
261 if (!bzlFileLoadLikeString.endsWith(".bzl")) {
262 bzlFileLoadLikeString = bzlFileLoadLikeString + ".bzl";
Dmitry Lomov51aafc12016-11-18 14:02:54 +0000263 }
264 }
265 SkylarkImport skylarkImport;
266 try {
dannarkf4b9ff42018-06-12 09:28:45 -0700267 skylarkImport =
268 SkylarkImports.create(
269 bzlFileLoadLikeString, /* repositoryMapping= */ ImmutableMap.of());
Dmitry Lomov51aafc12016-11-18 14:02:54 +0000270 } catch (SkylarkImportSyntaxException e) {
271 throw new ViewCreationFailedException(
272 String.format("Invalid aspect '%s': %s", aspect, e.getMessage()), e);
273 }
Dmitry Lomov0b832ce2015-10-20 10:03:14 +0000274
275 String skylarkFunctionName = aspect.substring(delimiterPosition + 1);
Dmitry Lomov01f85c02016-06-29 06:37:50 +0000276 for (TargetAndConfiguration targetSpec : topLevelTargetsWithConfigs) {
Klaus Aehligc27b4da2017-09-01 10:33:22 +0200277 aspectConfigurations.put(
278 Pair.of(targetSpec.getLabel(), aspect), targetSpec.getConfiguration());
Dmitry Lomove2033b12015-08-19 16:57:49 +0000279 aspectKeys.add(
Dmitry Lomov0b832ce2015-10-20 10:03:14 +0000280 AspectValue.createSkylarkAspectKey(
281 targetSpec.getLabel(),
Michael Staib04f6f242016-03-01 15:40:29 +0000282 // For invoking top-level aspects, use the top-level configuration for both the
283 // aspect and the base target while the top-level configuration is untrimmed.
284 targetSpec.getConfiguration(),
Dmitry Lomov0b832ce2015-10-20 10:03:14 +0000285 targetSpec.getConfiguration(),
Dmitry Lomov51aafc12016-11-18 14:02:54 +0000286 skylarkImport,
Dmitry Lomov0b832ce2015-10-20 10:03:14 +0000287 skylarkFunctionName));
Dmitry Lomove2033b12015-08-19 16:57:49 +0000288 }
289 } else {
Luis Fernando Pino Duquee82713d2016-04-26 16:22:38 +0000290 final NativeAspectClass aspectFactoryClass =
291 ruleClassProvider.getNativeAspectClassMap().get(aspect);
dslomov99ea6b42017-04-25 17:46:17 +0200292
Dmitry Lomov0b832ce2015-10-20 10:03:14 +0000293 if (aspectFactoryClass != null) {
Dmitry Lomov01f85c02016-06-29 06:37:50 +0000294 for (TargetAndConfiguration targetSpec : topLevelTargetsWithConfigs) {
Dmitry Lomovca9bfa42016-11-15 13:22:36 +0000295 // For invoking top-level aspects, use the top-level configuration for both the
296 // aspect and the base target while the top-level configuration is untrimmed.
297 BuildConfiguration configuration = targetSpec.getConfiguration();
Klaus Aehligc27b4da2017-09-01 10:33:22 +0200298 aspectConfigurations.put(Pair.of(targetSpec.getLabel(), aspect), configuration);
Dmitry Lomov0b832ce2015-10-20 10:03:14 +0000299 aspectKeys.add(
300 AspectValue.createAspectKey(
Dmitry Lomov8ff06452015-10-21 19:16:30 +0000301 targetSpec.getLabel(),
Dmitry Lomovca9bfa42016-11-15 13:22:36 +0000302 configuration,
dslomovfa50c3d2017-05-08 08:47:44 -0400303 new AspectDescriptor(aspectFactoryClass, AspectParameters.EMPTY),
Dmitry Lomovca9bfa42016-11-15 13:22:36 +0000304 configuration
305 ));
Dmitry Lomov0b832ce2015-10-20 10:03:14 +0000306 }
307 } else {
308 throw new ViewCreationFailedException("Aspect '" + aspect + "' is unknown");
309 }
Dmitry Lomove2033b12015-08-19 16:57:49 +0000310 }
311 }
312
Klaus Aehligc27b4da2017-09-01 10:33:22 +0200313 for (Pair<Label, String> target : aspectConfigurations.keys()) {
314 eventBus.post(
315 new AspectConfiguredEvent(
316 target.getFirst(), target.getSecond(), aspectConfigurations.get(target)));
317 }
318
Dmitry Lomove2033b12015-08-19 16:57:49 +0000319 SkyframeAnalysisResult skyframeAnalysisResult;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100320 try {
Dmitry Lomove2033b12015-08-19 16:57:49 +0000321 skyframeAnalysisResult =
322 skyframeBuildView.configureTargets(
juliexxiaf8b57c52017-12-21 11:37:46 -0800323 eventHandler, topLevelCtKeys, aspectKeys, eventBus, keepGoing, loadingPhaseThreads);
Kristina Chodorow94e27422016-08-25 14:17:30 +0000324 setArtifactRoots(skyframeAnalysisResult.getPackageRoots());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100325 } finally {
326 skyframeBuildView.clearInvalidatedConfiguredTargets();
327 }
328
Greg Estren7971e672016-06-01 23:22:48 +0000329 int numTargetsToAnalyze = topLevelTargetsWithConfigs.size();
Dmitry Lomove2033b12015-08-19 16:57:49 +0000330 int numSuccessful = skyframeAnalysisResult.getConfiguredTargets().size();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100331 if (0 < numSuccessful && numSuccessful < numTargetsToAnalyze) {
332 String msg = String.format("Analysis succeeded for only %d of %d top-level targets",
333 numSuccessful, numTargetsToAnalyze);
334 eventHandler.handle(Event.info(msg));
lberki998ec922017-09-05 10:08:25 +0200335 logger.info(msg);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100336 }
337
gregce45ae0962017-07-22 00:11:13 +0200338 Set<ConfiguredTarget> targetsToSkip =
janakr171a7eb2018-03-26 09:26:53 -0700339 new TopLevelConstraintSemantics(
340 skyframeExecutor.getPackageManager(),
341 input -> skyframeExecutor.getConfiguration(eventHandler, input),
342 eventHandler)
gregce21fdc482017-12-05 16:10:47 -0800343 .checkTargetEnvironmentRestrictions(skyframeAnalysisResult.getConfiguredTargets());
gregce45ae0962017-07-22 00:11:13 +0200344
Dmitry Lomove2033b12015-08-19 16:57:49 +0000345 AnalysisResult result =
346 createResult(
Ulf Adams5b9009b2015-09-24 09:52:53 +0000347 eventHandler,
Dmitry Lomove2033b12015-08-19 16:57:49 +0000348 loadingResult,
ulfjack940fbd72018-06-15 03:23:58 -0700349 configurations,
Dmitry Lomove2033b12015-08-19 16:57:49 +0000350 topLevelOptions,
351 viewOptions,
gregce45ae0962017-07-22 00:11:13 +0200352 skyframeAnalysisResult,
juliexxia693abdc2018-01-03 08:59:18 -0800353 targetsToSkip,
354 topLevelTargetsWithConfigs);
lberki998ec922017-09-05 10:08:25 +0200355 logger.info("Finished analysis");
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100356 return result;
357 }
358
Dmitry Lomove2033b12015-08-19 16:57:49 +0000359 private AnalysisResult createResult(
Klaus Aehlig777b30d2017-02-24 16:30:15 +0000360 ExtendedEventHandler eventHandler,
ulfjack12bb59f52018-07-25 07:19:24 -0700361 TargetPatternPhaseValue loadingResult,
ulfjack940fbd72018-06-15 03:23:58 -0700362 BuildConfigurationCollection configurations,
Dmitry Lomove2033b12015-08-19 16:57:49 +0000363 TopLevelArtifactContext topLevelOptions,
ulfjackb5daae32018-06-13 07:59:33 -0700364 AnalysisOptions viewOptions,
gregce45ae0962017-07-22 00:11:13 +0200365 SkyframeAnalysisResult skyframeAnalysisResult,
juliexxia693abdc2018-01-03 08:59:18 -0800366 Set<ConfiguredTarget> targetsToSkip,
367 List<TargetAndConfiguration> topLevelTargetsWithConfigs)
Klaus Aehlig777b30d2017-02-24 16:30:15 +0000368 throws InterruptedException {
ulfjack12bb59f52018-07-25 07:19:24 -0700369 Set<Label> testsToRun = loadingResult.getTestsToRunLabels();
gregce45ae0962017-07-22 00:11:13 +0200370 Set<ConfiguredTarget> configuredTargets =
371 Sets.newLinkedHashSet(skyframeAnalysisResult.getConfiguredTargets());
Googler92044992018-03-14 14:26:30 -0700372 ImmutableSet<AspectValue> aspects = ImmutableSet.copyOf(skyframeAnalysisResult.getAspects());
Dmitry Lomovcf6021a2016-08-09 16:49:32 +0000373
gregce45ae0962017-07-22 00:11:13 +0200374 Set<ConfiguredTarget> allTargetsToTest = null;
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100375 if (testsToRun != null) {
376 // Determine the subset of configured targets that are meant to be run as tests.
mjhalupkae253f832018-06-13 14:19:26 -0700377 allTargetsToTest = filterTestsByTargets(configuredTargets, testsToRun);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100378 }
379
janakr75bc18a2018-07-13 00:52:17 -0700380 SetMultimap<Artifact, Label> topLevelArtifactsToOwnerLabels = HashMultimap.create();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100381 Set<ConfiguredTarget> parallelTests = new HashSet<>();
382 Set<ConfiguredTarget> exclusiveTests = new HashSet<>();
Ulf Adamsfc154ae2015-08-31 12:32:14 +0000383
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100384 // build-info and build-changelist.
Ulf Adams5b9009b2015-09-24 09:52:53 +0000385 Collection<Artifact> buildInfoArtifacts =
386 skyframeExecutor.getWorkspaceStatusArtifacts(eventHandler);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100387 Preconditions.checkState(buildInfoArtifacts.size() == 2, buildInfoArtifacts);
janakr75bc18a2018-07-13 00:52:17 -0700388 addArtifactsWithNoOwner(buildInfoArtifacts, topLevelArtifactsToOwnerLabels);
Ulf Adamsfc154ae2015-08-31 12:32:14 +0000389
390 // Extra actions
mjhalupka7675dbd2018-01-29 12:46:58 -0800391 addExtraActionsIfRequested(
janakr75bc18a2018-07-13 00:52:17 -0700392 viewOptions, configuredTargets, aspects, topLevelArtifactsToOwnerLabels, eventHandler);
Ulf Adamsfc154ae2015-08-31 12:32:14 +0000393
394 // Coverage
janakr75bc18a2018-07-13 00:52:17 -0700395 NestedSet<Artifact> baselineCoverageArtifacts =
396 getBaselineCoverageArtifacts(configuredTargets, topLevelArtifactsToOwnerLabels);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100397 if (coverageReportActionFactory != null) {
Googlercdc90e92015-02-11 00:18:07 +0000398 CoverageReportActionsWrapper actionsWrapper;
janakr573807d2018-01-11 14:02:35 -0800399 actionsWrapper =
400 coverageReportActionFactory.createCoverageReportActionsWrapper(
401 eventHandler,
402 directories,
403 allTargetsToTest,
404 baselineCoverageArtifacts,
405 getArtifactFactory(),
elenairina9438d102018-07-06 03:06:21 -0700406 CoverageReportValue.COVERAGE_REPORT_KEY,
407 loadingResult.getWorkspaceName());
Googlercdc90e92015-02-11 00:18:07 +0000408 if (actionsWrapper != null) {
Rumou Duan33bab462016-04-25 17:55:12 +0000409 ImmutableList<ActionAnalysisMetadata> actions = actionsWrapper.getActions();
Googlercdc90e92015-02-11 00:18:07 +0000410 skyframeExecutor.injectCoverageReportData(actions);
janakr75bc18a2018-07-13 00:52:17 -0700411 addArtifactsWithNoOwner(
412 actionsWrapper.getCoverageOutputs(), topLevelArtifactsToOwnerLabels);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100413 }
414 }
415
Ulf Adamsfc154ae2015-08-31 12:32:14 +0000416 // Tests. This must come last, so that the exclusive tests are scheduled after everything else.
mjhalupka7675dbd2018-01-29 12:46:58 -0800417 scheduleTestsIfRequested(
418 parallelTests,
419 exclusiveTests,
420 topLevelOptions,
421 allTargetsToTest,
422 skyframeExecutor,
423 eventHandler);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100424
Janak Ramakrishnan6f9d7d12016-08-09 08:45:34 +0000425 String error = createErrorMessage(loadingResult, skyframeAnalysisResult);
Eric Fellheimer9c1e12f2015-08-06 20:23:55 +0000426
Ulf Adams53abece2016-02-03 08:30:07 +0000427 final WalkableGraph graph = skyframeAnalysisResult.getWalkableGraph();
Janak Ramakrishnan3c0adb22016-08-15 21:54:55 +0000428 final ActionGraph actionGraph =
429 new ActionGraph() {
430 @Nullable
431 @Override
432 public ActionAnalysisMetadata getGeneratingAction(Artifact artifact) {
433 ArtifactOwner artifactOwner = artifact.getArtifactOwner();
434 if (artifactOwner instanceof ActionLookupValue.ActionLookupKey) {
janakr573807d2018-01-11 14:02:35 -0800435 SkyKey key = (ActionLookupValue.ActionLookupKey) artifactOwner;
Janak Ramakrishnan3c0adb22016-08-15 21:54:55 +0000436 ActionLookupValue val;
437 try {
438 val = (ActionLookupValue) graph.getValue(key);
439 } catch (InterruptedException e) {
440 throw new IllegalStateException(
441 "Interruption not expected from this graph: " + key, e);
442 }
janakr93e3eea2017-03-30 22:09:37 +0000443 return val == null ? null : val.getGeneratingActionDangerousReadJavadoc(artifact);
Janak Ramakrishnan3c0adb22016-08-15 21:54:55 +0000444 }
445 return null;
446 }
447 };
Dmitry Lomove2033b12015-08-19 16:57:49 +0000448 return new AnalysisResult(
ulfjack940fbd72018-06-15 03:23:58 -0700449 configurations,
Dmitry Lomove2033b12015-08-19 16:57:49 +0000450 configuredTargets,
Dmitry Lomovcf6021a2016-08-09 16:49:32 +0000451 aspects,
Dmitry Lomove2033b12015-08-19 16:57:49 +0000452 allTargetsToTest,
gregce45ae0962017-07-22 00:11:13 +0200453 targetsToSkip,
Dmitry Lomove2033b12015-08-19 16:57:49 +0000454 error,
455 actionGraph,
janakr75bc18a2018-07-13 00:52:17 -0700456 topLevelArtifactsToOwnerLabels,
Dmitry Lomove2033b12015-08-19 16:57:49 +0000457 parallelTests,
458 exclusiveTests,
Marian Lobur86bd4fd2015-09-16 10:01:38 +0000459 topLevelOptions,
Kristina Chodorow36a6c172016-07-22 15:18:07 +0000460 skyframeAnalysisResult.getPackageRoots(),
juliexxia693abdc2018-01-03 08:59:18 -0800461 loadingResult.getWorkspaceName(),
462 topLevelTargetsWithConfigs);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100463 }
464
janakr75bc18a2018-07-13 00:52:17 -0700465 private static void addArtifactsWithNoOwner(
466 Collection<Artifact> artifacts, SetMultimap<Artifact, Label> topLevelArtifactsToOwnerLabels) {
467 artifacts.forEach((a) -> topLevelArtifactsToOwnerLabels.put(a, a.getOwnerLabel()));
468 }
469
Janak Ramakrishnan6f9d7d12016-08-09 08:45:34 +0000470 @Nullable
471 public static String createErrorMessage(
ulfjack12bb59f52018-07-25 07:19:24 -0700472 TargetPatternPhaseValue loadingResult,
473 @Nullable SkyframeAnalysisResult skyframeAnalysisResult) {
474 return loadingResult.hasError()
Janak Ramakrishnan6f9d7d12016-08-09 08:45:34 +0000475 ? "command succeeded, but there were errors parsing the target pattern"
ulfjack12bb59f52018-07-25 07:19:24 -0700476 : loadingResult.hasPostExpansionError()
Janak Ramakrishnan6f9d7d12016-08-09 08:45:34 +0000477 || (skyframeAnalysisResult != null && skyframeAnalysisResult.hasLoadingError())
478 ? "command succeeded, but there were loading phase errors"
479 : (skyframeAnalysisResult != null && skyframeAnalysisResult.hasAnalysisError())
480 ? "command succeeded, but not all targets were analyzed"
481 : null;
482 }
483
Lukacs Berki6916be22015-02-19 13:36:06 +0000484 private static NestedSet<Artifact> getBaselineCoverageArtifacts(
janakr75bc18a2018-07-13 00:52:17 -0700485 Collection<ConfiguredTarget> configuredTargets,
486 SetMultimap<Artifact, Label> topLevelArtifactsToOwnerLabels) {
Lukacs Berki6916be22015-02-19 13:36:06 +0000487 NestedSetBuilder<Artifact> baselineCoverageArtifacts = NestedSetBuilder.stableOrder();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100488 for (ConfiguredTarget target : configuredTargets) {
Ulf Adamsfc154ae2015-08-31 12:32:14 +0000489 InstrumentedFilesProvider provider = target.getProvider(InstrumentedFilesProvider.class);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100490 if (provider != null) {
janakr75bc18a2018-07-13 00:52:17 -0700491 TopLevelArtifactHelper.addArtifactsWithOwnerLabel(
492 provider.getBaselineCoverageArtifacts(),
493 null,
494 target.getLabel(),
495 topLevelArtifactsToOwnerLabels);
Ulf Adamsfc154ae2015-08-31 12:32:14 +0000496 baselineCoverageArtifacts.addTransitive(provider.getBaselineCoverageArtifacts());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100497 }
498 }
Lukacs Berki6916be22015-02-19 13:36:06 +0000499 return baselineCoverageArtifacts.build();
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100500 }
501
mjhalupka7675dbd2018-01-29 12:46:58 -0800502 private void addExtraActionsIfRequested(
ulfjackb5daae32018-06-13 07:59:33 -0700503 AnalysisOptions viewOptions,
Dmitry Lomovcf6021a2016-08-09 16:49:32 +0000504 Collection<ConfiguredTarget> configuredTargets,
505 Collection<AspectValue> aspects,
janakr75bc18a2018-07-13 00:52:17 -0700506 SetMultimap<Artifact, Label> artifactsToTopLevelLabelsMap,
mjhalupka7675dbd2018-01-29 12:46:58 -0800507 ExtendedEventHandler eventHandler) {
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100508 RegexFilter filter = viewOptions.extraActionFilter;
Carmi Grushkodf9e5e12016-11-08 23:07:57 +0000509 for (ConfiguredTarget target : configuredTargets) {
510 ExtraActionArtifactsProvider provider =
511 target.getProvider(ExtraActionArtifactsProvider.class);
512 if (provider != null) {
513 if (viewOptions.extraActionTopLevelOnly) {
lberki9137a752017-06-09 12:49:10 +0200514 // Collect all aspect-classes that topLevel might inject.
515 Set<AspectClass> aspectClasses = new HashSet<>();
mjhalupka7675dbd2018-01-29 12:46:58 -0800516 Target actualTarget = null;
517 try {
518 actualTarget =
519 skyframeExecutor.getPackageManager().getTarget(eventHandler, target.getLabel());
520 } catch (NoSuchPackageException | NoSuchTargetException | InterruptedException e) {
521 eventHandler.handle(Event.error(""));
522 }
523 for (Attribute attr : actualTarget.getAssociatedRule().getAttributes()) {
lberki9137a752017-06-09 12:49:10 +0200524 aspectClasses.addAll(attr.getAspectClasses());
525 }
janakr75bc18a2018-07-13 00:52:17 -0700526 TopLevelArtifactHelper.addArtifactsWithOwnerLabel(
527 provider.getExtraActionArtifacts(),
528 filter,
529 target.getLabel(),
530 artifactsToTopLevelLabelsMap);
lberki9137a752017-06-09 12:49:10 +0200531 if (!aspectClasses.isEmpty()) {
janakr75bc18a2018-07-13 00:52:17 -0700532 TopLevelArtifactHelper.addArtifactsWithOwnerLabel(
533 filterTransitiveExtraActions(provider, aspectClasses),
534 filter,
535 target.getLabel(),
536 artifactsToTopLevelLabelsMap);
Carmi Grushkodf9e5e12016-11-08 23:07:57 +0000537 }
Carmi Grushkodf9e5e12016-11-08 23:07:57 +0000538 } else {
janakr75bc18a2018-07-13 00:52:17 -0700539 TopLevelArtifactHelper.addArtifactsWithOwnerLabel(
540 provider.getTransitiveExtraActionArtifacts(),
541 filter,
542 target.getLabel(),
543 artifactsToTopLevelLabelsMap);
Carmi Grushkodf9e5e12016-11-08 23:07:57 +0000544 }
545 }
546 }
janakr75bc18a2018-07-13 00:52:17 -0700547 for (AspectValue aspect : aspects) {
548 ExtraActionArtifactsProvider provider =
549 aspect.getConfiguredAspect().getProvider(ExtraActionArtifactsProvider.class);
550 if (provider != null) {
551 if (viewOptions.extraActionTopLevelOnly) {
552 TopLevelArtifactHelper.addArtifactsWithOwnerLabel(
553 provider.getExtraActionArtifacts(),
554 filter,
555 aspect.getLabel(),
556 artifactsToTopLevelLabelsMap);
557 } else {
558 TopLevelArtifactHelper.addArtifactsWithOwnerLabel(
559 provider.getTransitiveExtraActionArtifacts(),
560 filter,
561 aspect.getLabel(),
562 artifactsToTopLevelLabelsMap);
563 }
564 }
565 }
Carmi Grushkodf9e5e12016-11-08 23:07:57 +0000566 }
567
568 /**
569 * Returns a list of actions from 'provider' that were registered by an aspect from
570 * 'aspectClasses'. All actions in 'provider' are considered - both direct and transitive.
571 */
572 private ImmutableList<Artifact> filterTransitiveExtraActions(
573 ExtraActionArtifactsProvider provider, Set<AspectClass> aspectClasses) {
574 ImmutableList.Builder<Artifact> artifacts = ImmutableList.builder();
575 // Add to 'artifacts' all extra-actions which were registered by aspects which 'topLevel'
576 // might have injected.
Carmi Grushkobabd4852016-11-18 17:58:09 +0000577 for (Artifact artifact : provider.getTransitiveExtraActionArtifacts()) {
578 ArtifactOwner owner = artifact.getArtifactOwner();
579 if (owner instanceof AspectKey) {
580 if (aspectClasses.contains(((AspectKey) owner).getAspectClass())) {
581 artifacts.add(artifact);
Carmi Grushkodf9e5e12016-11-08 23:07:57 +0000582 }
583 }
584 }
585 return artifacts.build();
586 }
587
mjhalupka7675dbd2018-01-29 12:46:58 -0800588 private static void scheduleTestsIfRequested(
589 Collection<ConfiguredTarget> targetsToTest,
590 Collection<ConfiguredTarget> targetsToTestExclusive,
591 TopLevelArtifactContext topLevelOptions,
592 Collection<ConfiguredTarget> allTestTargets,
593 SkyframeExecutor skyframeExecutor,
594 ExtendedEventHandler eventHandler)
595 throws InterruptedException {
Lukacs Berki3f4d4e92015-02-24 10:28:26 +0000596 Set<String> outputGroups = topLevelOptions.outputGroups();
dslomov69c45f82017-12-14 11:15:43 -0500597 if (!outputGroups.contains(OutputGroupInfo.FILES_TO_COMPILE)
598 && !outputGroups.contains(OutputGroupInfo.COMPILATION_PREREQUISITES)
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100599 && allTestTargets != null) {
mjhalupka7675dbd2018-01-29 12:46:58 -0800600 scheduleTests(
601 targetsToTest,
602 targetsToTestExclusive,
603 allTestTargets,
604 topLevelOptions.runTestsExclusively(),
605 skyframeExecutor,
606 eventHandler);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100607 }
608 }
609
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100610 /**
611 * Returns set of artifacts representing test results, writing into targetsToTest and
612 * targetsToTestExclusive.
613 */
mjhalupka7675dbd2018-01-29 12:46:58 -0800614 private static void scheduleTests(
615 Collection<ConfiguredTarget> targetsToTest,
616 Collection<ConfiguredTarget> targetsToTestExclusive,
617 Collection<ConfiguredTarget> allTestTargets,
618 boolean isExclusive,
619 SkyframeExecutor skyframeExecutor,
620 ExtendedEventHandler eventHandler)
621 throws InterruptedException {
622 for (ConfiguredTarget configuredTarget : allTestTargets) {
623 Target target = null;
624 try {
625 target =
626 skyframeExecutor
627 .getPackageManager()
628 .getTarget(eventHandler, configuredTarget.getLabel());
629 } catch (NoSuchTargetException | NoSuchPackageException e) {
630 eventHandler.handle(Event.error("Failed to get target when scheduling tests"));
631 continue;
632 }
633 if (target instanceof Rule) {
634 boolean exclusive = isExclusive || TargetUtils.isExclusiveTestRule((Rule) target);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100635 Collection<ConfiguredTarget> testCollection = exclusive
636 ? targetsToTestExclusive
637 : targetsToTest;
mjhalupka7675dbd2018-01-29 12:46:58 -0800638 testCollection.add(configuredTarget);
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100639 }
640 }
641 }
642
Greg Estren7971e672016-06-01 23:22:48 +0000643 /**
Ulf Adamsa385d3e2015-09-28 13:21:45 +0000644 * Sets the possible artifact roots in the artifact factory. This allows the factory to resolve
645 * paths with unknown roots to artifacts.
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100646 */
ulfjack865b6212018-06-14 03:41:55 -0700647 private void setArtifactRoots(PackageRoots packageRoots) {
janakrae323982017-09-29 21:11:53 +0200648 getArtifactFactory().setPackageRoots(packageRoots.getPackageRootLookup());
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100649 }
650
651 /**
Ulf Adams9c505cb2015-09-28 13:38:14 +0000652 * Tests and clears the current thread's pending "interrupted" status, and
653 * throws InterruptedException iff it was set.
654 */
ulfjack865b6212018-06-14 03:41:55 -0700655 private final void pollInterruptedStatus() throws InterruptedException {
Ulf Adams9c505cb2015-09-28 13:38:14 +0000656 if (Thread.interrupted()) {
657 throw new InterruptedException();
658 }
659 }
Han-Wen Nienhuysd08b27f2015-02-25 16:45:20 +0100660}