blob: 7d90a019b368019a11003e44ce922558260ec6fd [file] [log] [blame]
Janak Ramakrishnane72d5222015-02-26 17:09:18 +00001// Copyright 2015 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.query2;
15
Janak Ramakrishnan36858732015-06-17 16:45:47 +000016import com.google.common.base.Function;
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +000017import com.google.common.base.Functions;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000018import com.google.common.base.Preconditions;
19import com.google.common.base.Predicate;
Janak Ramakrishnanb735d2f2015-06-16 17:46:36 +000020import com.google.common.base.Predicates;
Janak Ramakrishnanf6f0fcc2015-06-19 20:24:52 +000021import com.google.common.collect.ArrayListMultimap;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000022import com.google.common.collect.Collections2;
Janak Ramakrishnancda5b662015-06-18 23:46:36 +000023import com.google.common.collect.ImmutableList;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000024import com.google.common.collect.ImmutableMap;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000025import com.google.common.collect.ImmutableSet;
26import com.google.common.collect.Iterables;
27import com.google.common.collect.Maps;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000028import com.google.common.collect.Multimap;
Miguel Alcon Pinto933c13a2015-09-16 18:37:45 +000029import com.google.common.collect.Sets;
Lukacs Berki6e91eb92015-09-21 09:12:37 +000030import com.google.devtools.build.lib.cmdline.Label;
Lukacs Berkia6434362015-09-15 13:56:14 +000031import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
Kristina Chodorow73fa2032015-08-28 17:57:46 +000032import com.google.devtools.build.lib.cmdline.PackageIdentifier;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000033import com.google.devtools.build.lib.cmdline.TargetParsingException;
Mark Schallerb889cf32015-03-17 20:55:30 +000034import com.google.devtools.build.lib.cmdline.TargetPattern;
David Chend6594262015-08-21 09:39:07 +000035import com.google.devtools.build.lib.collect.CompactHashSet;
Mark Schaller895b3d22015-03-23 14:27:26 +000036import com.google.devtools.build.lib.events.Event;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000037import com.google.devtools.build.lib.events.EventHandler;
38import com.google.devtools.build.lib.graph.Digraph;
Janak Ramakrishnan0a4c6e42015-09-17 00:37:58 +000039import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000040import com.google.devtools.build.lib.packages.NoSuchTargetException;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000041import com.google.devtools.build.lib.packages.NoSuchThingException;
42import com.google.devtools.build.lib.packages.Package;
43import com.google.devtools.build.lib.packages.Rule;
44import com.google.devtools.build.lib.packages.Target;
Mark Schallerb889cf32015-03-17 20:55:30 +000045import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000046import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
Eric Fellheimera39f8a92015-07-28 19:11:23 +000047import com.google.devtools.build.lib.profiler.Profiler;
Janak Ramakrishnan643063d2015-06-25 16:21:49 +000048import com.google.devtools.build.lib.query2.engine.AllRdepsFunction;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000049import com.google.devtools.build.lib.query2.engine.QueryEvalResult;
50import com.google.devtools.build.lib.query2.engine.QueryException;
51import com.google.devtools.build.lib.query2.engine.QueryExpression;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000052import com.google.devtools.build.lib.skyframe.FileValue;
Mark Schallerb889cf32015-03-17 20:55:30 +000053import com.google.devtools.build.lib.skyframe.GraphBackedRecursivePackageProvider;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000054import com.google.devtools.build.lib.skyframe.PackageLookupValue;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000055import com.google.devtools.build.lib.skyframe.PackageValue;
Mark Schallerd7311e02015-07-07 16:36:09 +000056import com.google.devtools.build.lib.skyframe.PrepareDepsOfPatternsValue;
Mark Schallerb889cf32015-03-17 20:55:30 +000057import com.google.devtools.build.lib.skyframe.RecursivePackageProviderBackedTargetPatternResolver;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000058import com.google.devtools.build.lib.skyframe.SkyFunctions;
59import com.google.devtools.build.lib.skyframe.TargetPatternValue;
Mark Schallerd7311e02015-07-07 16:36:09 +000060import com.google.devtools.build.lib.skyframe.TargetPatternValue.TargetPatternKey;
Mark Schaller8ff5b3c2015-07-29 17:32:11 +000061import com.google.devtools.build.lib.skyframe.TransitiveTraversalValue;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000062import com.google.devtools.build.lib.vfs.PathFragment;
63import com.google.devtools.build.lib.vfs.RootedPath;
Mark Schallerd7311e02015-07-07 16:36:09 +000064import com.google.devtools.build.skyframe.EvaluationResult;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000065import com.google.devtools.build.skyframe.SkyFunctionName;
66import com.google.devtools.build.skyframe.SkyKey;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000067import com.google.devtools.build.skyframe.SkyValue;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000068import com.google.devtools.build.skyframe.WalkableGraph;
69import com.google.devtools.build.skyframe.WalkableGraph.WalkableGraphFactory;
70
71import java.util.ArrayDeque;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000072import java.util.Collection;
73import java.util.Deque;
74import java.util.HashMap;
75import java.util.HashSet;
76import java.util.Iterator;
77import java.util.LinkedHashSet;
78import java.util.List;
79import java.util.Map;
80import java.util.Set;
Eric Fellheimera39f8a92015-07-28 19:11:23 +000081import java.util.logging.Logger;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000082
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +000083import javax.annotation.Nullable;
84
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000085/**
86 * {@link AbstractBlazeQueryEnvironment} that introspects the Skyframe graph to find forward and
87 * reverse edges. Results obtained by calling {@link #evaluateQuery} are not guaranteed to be in
88 * any particular order. As well, this class eagerly loads the full transitive closure of targets,
89 * even if the full closure isn't needed.
90 */
91public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> {
Miguel Alcon Pinto45820872015-09-11 19:57:47 +000092
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000093 private WalkableGraph graph;
94
Mark Schallerd7311e02015-07-07 16:36:09 +000095 private ImmutableList<TargetPatternKey> universeTargetPatternKeys;
96
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000097 private final BlazeTargetAccessor accessor = new BlazeTargetAccessor(this);
98 private final int loadingPhaseThreads;
99 private final WalkableGraphFactory graphFactory;
100 private final List<String> universeScope;
101 private final String parserPrefix;
Mark Schallerb889cf32015-03-17 20:55:30 +0000102 private final PathPackageLocator pkgPath;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000103
Eric Fellheimera39f8a92015-07-28 19:11:23 +0000104 private static final Logger LOG = Logger.getLogger(SkyQueryEnvironment.class.getName());
105
Miguel Alcon Pintob45e2622015-08-21 18:31:23 +0000106 private static final Function<Target, Label> TARGET_LABEL_FUNCTION =
107 new Function<Target, Label>() {
108
109 @Override
110 public Label apply(Target target) {
111 return target.getLabel();
112 }
113 };
114
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000115 public SkyQueryEnvironment(boolean keepGoing, boolean strictScope, int loadingPhaseThreads,
116 Predicate<Label> labelFilter,
117 EventHandler eventHandler,
118 Set<Setting> settings,
119 Iterable<QueryFunction> extraFunctions, String parserPrefix,
120 WalkableGraphFactory graphFactory,
Mark Schallerb889cf32015-03-17 20:55:30 +0000121 List<String> universeScope, PathPackageLocator pkgPath) {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000122 super(keepGoing, strictScope, labelFilter,
123 eventHandler,
124 settings,
125 extraFunctions);
126 this.loadingPhaseThreads = loadingPhaseThreads;
127 this.graphFactory = graphFactory;
Mark Schallerb889cf32015-03-17 20:55:30 +0000128 this.pkgPath = pkgPath;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000129 this.universeScope = Preconditions.checkNotNull(universeScope);
130 this.parserPrefix = parserPrefix;
131 Preconditions.checkState(!universeScope.isEmpty(),
132 "No queries can be performed with an empty universe");
133 }
134
135 private void init() throws InterruptedException {
Eric Fellheimera39f8a92015-07-28 19:11:23 +0000136 long startTime = Profiler.nanoTimeMaybe();
Mark Schallerd7311e02015-07-07 16:36:09 +0000137 EvaluationResult<SkyValue> result =
138 graphFactory.prepareAndGet(universeScope, loadingPhaseThreads, eventHandler);
139 graph = result.getWalkableGraph();
Eric Fellheimera39f8a92015-07-28 19:11:23 +0000140 long duration = Profiler.nanoTimeMaybe() - startTime;
141 if (duration > 0) {
142 LOG.info("Spent " + (duration / 1000 / 1000) + " ms on evaluation and walkable graph");
143 }
Mark Schallerd7311e02015-07-07 16:36:09 +0000144
Mark Schaller28a676092015-08-27 18:11:04 +0000145 // The prepareAndGet call above evaluates a single PrepareDepsOfPatterns SkyKey.
146 // We expect to see either a single successfully evaluated value or a cycle in the result.
147 Collection<SkyValue> values = result.values();
148 if (!values.isEmpty()) {
149 Preconditions.checkState(values.size() == 1, "Universe query \"%s\" returned multiple"
150 + " values unexpectedly (%s values in result)", universeScope, values.size());
Mark Schallerd7311e02015-07-07 16:36:09 +0000151 PrepareDepsOfPatternsValue prepareDepsOfPatternsValue =
152 (PrepareDepsOfPatternsValue) Iterables.getOnlyElement(values);
153 universeTargetPatternKeys = prepareDepsOfPatternsValue.getTargetPatternKeys();
154 } else {
Mark Schaller28a676092015-08-27 18:11:04 +0000155 // No values in the result, so there must be an error. We expect the error to be a cycle.
156 boolean foundCycle = !Iterables.isEmpty(result.getError().getCycleInfo());
157 Preconditions.checkState(foundCycle, "Universe query \"%s\" failed with non-cycle error: %s",
158 universeScope, result.getError());
Mark Schallerd7311e02015-07-07 16:36:09 +0000159 universeTargetPatternKeys = ImmutableList.of();
160 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000161 }
162
163 @Override
164 public QueryEvalResult<Target> evaluateQuery(QueryExpression expr)
Janak Ramakrishnan89907392015-07-08 21:43:31 +0000165 throws QueryException, InterruptedException {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000166 // Some errors are reported as QueryExceptions and others as ERROR events (if --keep_going). The
167 // result is set to have an error iff there were errors emitted during the query, so we reset
168 // errors here.
169 eventHandler.resetErrors();
Janak Ramakrishnan89907392015-07-08 21:43:31 +0000170 init();
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000171 return super.evaluateQuery(expr);
172 }
173
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000174 private Map<Target, Collection<Target>> makeTargetsMap(Map<SkyKey, Iterable<SkyKey>> input) {
175 ImmutableMap.Builder<Target, Collection<Target>> result = ImmutableMap.builder();
Miguel Alcon Pinto933c13a2015-09-16 18:37:45 +0000176
177 Map<SkyKey, Target> allTargets = makeTargetsWithAssociations(
178 Sets.newHashSet(Iterables.concat(input.values())));
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000179
180 for (Map.Entry<SkyKey, Target> entry : makeTargetsWithAssociations(input.keySet()).entrySet()) {
Miguel Alcon Pinto933c13a2015-09-16 18:37:45 +0000181 Iterable<SkyKey> skyKeys = input.get(entry.getKey());
182 Set<Target> targets = CompactHashSet.createWithExpectedSize(Iterables.size(skyKeys));
183 for (SkyKey key : skyKeys) {
184 Target target = allTargets.get(key);
185 if (target != null) {
186 targets.add(target);
187 }
188 }
189 result.put(entry.getValue(), targets);
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000190 }
191 return result.build();
192 }
193
194 private Map<Target, Collection<Target>> getRawFwdDeps(Iterable<Target> targets) {
195 return makeTargetsMap(graph.getDirectDeps(makeKeys(targets)));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000196 }
197
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000198 private Map<Target, Collection<Target>> getRawReverseDeps(Iterable<Target> targets) {
199 return makeTargetsMap(graph.getReverseDeps(makeKeys(targets)));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000200 }
201
202 private Set<Label> getAllowedDeps(Rule rule) {
Miguel Alcon Pinto4ffe28d2015-08-19 14:29:02 +0000203 Set<Label> allowedLabels = new HashSet<>(rule.getTransitions(dependencyFilter).values());
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000204 allowedLabels.addAll(rule.getVisibility().getDependencyLabels());
Marian Loburfdd788e2015-03-25 09:36:28 +0000205 // We should add deps from aspects, otherwise they are going to be filtered out.
206 allowedLabels.addAll(rule.getAspectLabelsSuperset(dependencyFilter));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000207 return allowedLabels;
208 }
209
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000210 private Collection<Target> filterFwdDeps(Target target, Collection<Target> rawFwdDeps) {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000211 if (!(target instanceof Rule)) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000212 return rawFwdDeps;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000213 }
214 final Set<Label> allowedLabels = getAllowedDeps((Rule) target);
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000215 return Collections2.filter(rawFwdDeps,
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000216 new Predicate<Target>() {
217 @Override
218 public boolean apply(Target target) {
219 return allowedLabels.contains(target.getLabel());
220 }
221 });
222 }
223
Nathan Harmata3fae3662015-04-22 20:10:48 +0000224 @Override
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000225 public Collection<Target> getFwdDeps(Iterable<Target> targets) {
226 Set<Target> result = new HashSet<>();
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000227 for (Map.Entry<Target, Collection<Target>> entry : getRawFwdDeps(targets).entrySet()) {
228 result.addAll(filterFwdDeps(entry.getKey(), entry.getValue()));
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000229 }
230 return result;
231 }
232
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000233 @Override
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000234 public Collection<Target> getReverseDeps(Iterable<Target> targets) {
Miguel Alcon Pintob45e2622015-08-21 18:31:23 +0000235 Set<Target> result = CompactHashSet.create();
236 Map<Target, Collection<Target>> rawReverseDeps = getRawReverseDeps(targets);
237
238 CompactHashSet<Target> visited = CompactHashSet.create();
239
240 Set<Label> keys = CompactHashSet.create(Collections2.transform(rawReverseDeps.keySet(),
241 TARGET_LABEL_FUNCTION));
242 for (Collection<Target> parentCollection : rawReverseDeps.values()) {
243 for (Target parent : parentCollection) {
244 if (visited.add(parent)) {
245 if (parent instanceof Rule) {
246 for (Label label : getAllowedDeps((Rule) parent)) {
247 if (keys.contains(label)) {
248 result.add(parent);
249 }
250 }
251 } else {
252 result.add(parent);
253 }
254 }
255 }
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000256 }
257 return result;
258 }
259
260 @Override
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000261 public Set<Target> getTransitiveClosure(Set<Target> targets) {
262 Set<Target> visited = new HashSet<>();
Janak Ramakrishnanb735d2f2015-06-16 17:46:36 +0000263 Collection<Target> current = targets;
264 while (!current.isEmpty()) {
265 Collection<Target> toVisit = Collections2.filter(current,
266 Predicates.not(Predicates.in(visited)));
267 current = getFwdDeps(toVisit);
268 visited.addAll(toVisit);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000269 }
Janak Ramakrishnanb735d2f2015-06-16 17:46:36 +0000270 return ImmutableSet.copyOf(visited);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000271 }
272
273 // Implemented with a breadth-first search.
274 @Override
275 public Set<Target> getNodesOnPath(Target from, Target to) {
276 // Tree of nodes visited so far.
277 Map<Target, Target> nodeToParent = new HashMap<>();
278 // Contains all nodes left to visit in a (LIFO) stack.
279 Deque<Target> toVisit = new ArrayDeque<>();
280 toVisit.add(from);
281 nodeToParent.put(from, null);
282 while (!toVisit.isEmpty()) {
283 Target current = toVisit.removeFirst();
284 if (to.equals(current)) {
285 return ImmutableSet.copyOf(Digraph.getPathToTreeNode(nodeToParent, to));
286 }
Janak Ramakrishnancda5b662015-06-18 23:46:36 +0000287 for (Target dep : getFwdDeps(ImmutableList.of(current))) {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000288 if (!nodeToParent.containsKey(dep)) {
289 nodeToParent.put(dep, current);
290 toVisit.addFirst(dep);
291 }
292 }
293 }
294 // Note that the only current caller of this method checks first to see if there is a path
295 // before calling this method. It is not clear what the return value should be here.
296 return null;
297 }
298
299 @Override
300 public Set<Target> getTargetsMatchingPattern(QueryExpression owner, String pattern)
301 throws QueryException {
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000302 Set<Target> targets = new LinkedHashSet<>(resolvedTargetPatterns.get(pattern));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000303
304 // Sets.filter would be more convenient here, but can't deal with exceptions.
305 Iterator<Target> targetIterator = targets.iterator();
306 while (targetIterator.hasNext()) {
307 Target target = targetIterator.next();
308 if (!validateScope(target.getLabel(), strictScope)) {
309 targetIterator.remove();
310 }
311 }
312 return targets;
313 }
314
315 @Override
316 public Set<Target> getBuildFiles(QueryExpression caller, Set<Target> nodes)
317 throws QueryException {
318 Set<Target> dependentFiles = new LinkedHashSet<>();
319 Set<Package> seenPackages = new HashSet<>();
320 // Keep track of seen labels, to avoid adding a fake subinclude label that also exists as a
321 // real target.
322 Set<Label> seenLabels = new HashSet<>();
323
324 // Adds all the package definition files (BUILD files and build
325 // extensions) for package "pkg", to "buildfiles".
326 for (Target x : nodes) {
327 Package pkg = x.getPackage();
328 if (seenPackages.add(pkg)) {
329 addIfUniqueLabel(pkg.getBuildFile(), seenLabels, dependentFiles);
330 for (Label subinclude
331 : Iterables.concat(pkg.getSubincludeLabels(), pkg.getSkylarkFileDependencies())) {
332 addIfUniqueLabel(getSubincludeTarget(subinclude, pkg), seenLabels, dependentFiles);
333
334 // Also add the BUILD file of the subinclude.
335 try {
336 addIfUniqueLabel(getSubincludeTarget(
337 subinclude.getLocalTargetLabel("BUILD"), pkg), seenLabels, dependentFiles);
Lukacs Berkia6434362015-09-15 13:56:14 +0000338 } catch (LabelSyntaxException e) {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000339 throw new AssertionError("BUILD should always parse as a target name", e);
340 }
341 }
342 }
343 }
344 return dependentFiles;
345 }
346
347 private static void addIfUniqueLabel(Target node, Set<Label> labels, Set<Target> nodes) {
348 if (labels.add(node.getLabel())) {
349 nodes.add(node);
350 }
351 }
352
353 private static Target getSubincludeTarget(final Label label, Package pkg) {
Mark Schallerb817e3f2015-06-04 17:14:18 +0000354 return new FakeSubincludeTarget(label, pkg);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000355 }
356
357 @Override
358 public TargetAccessor<Target> getAccessor() {
359 return accessor;
360 }
361
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000362 private SkyKey getPackageKeyAndValidateLabel(Label label) throws QueryException {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000363 // Can't use strictScope here because we are expecting a target back.
364 validateScope(label, true);
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000365 return PackageValue.key(label.getPackageIdentifier());
366 }
367
368 @Override
369 public Target getTarget(Label label) throws TargetNotFoundException, QueryException {
370 SkyKey packageKey = getPackageKeyAndValidateLabel(label);
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000371 if (!graph.exists(packageKey)) {
372 throw new QueryException(packageKey + " does not exist in graph");
373 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000374 try {
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000375 PackageValue packageValue = (PackageValue) graph.getValue(packageKey);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000376 if (packageValue != null) {
Janak Ramakrishnan0a4c6e42015-09-17 00:37:58 +0000377 Package pkg = packageValue.getPackage();
378 if (pkg.containsErrors()) {
379 throw new BuildFileContainsErrorsException(label.getPackageIdentifier());
380 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000381 return packageValue.getPackage().getTarget(label.getName());
382 } else {
383 throw (NoSuchThingException) Preconditions.checkNotNull(
384 graph.getException(packageKey), label);
385 }
386 } catch (NoSuchThingException e) {
387 throw new TargetNotFoundException(e);
388 }
389 }
390
391 @Override
392 public void buildTransitiveClosure(QueryExpression caller, Set<Target> targets, int maxDepth)
393 throws QueryException {
394 // Everything has already been loaded, so here we just check for errors so that we can
395 // pre-emptively throw/report if needed.
Janak Ramakrishnanf6f0fcc2015-06-19 20:24:52 +0000396 for (Map.Entry<SkyKey, Exception> entry :
397 graph.getMissingAndExceptions(makeKeys(targets)).entrySet()) {
398 if (entry.getValue() == null) {
399 throw new QueryException(entry.getKey().argument() + " does not exist in graph");
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000400 }
Janak Ramakrishnanf6f0fcc2015-06-19 20:24:52 +0000401 reportBuildFileError(caller, entry.getValue().getMessage());
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000402 }
403 }
404
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000405 private Set<Target> filterTargetsNotInGraph(Set<Target> targets) {
406 Map<Target, SkyKey> map = Maps.toMap(targets, TARGET_TO_SKY_KEY);
Miguel Alcon Pinto45820872015-09-11 19:57:47 +0000407 Set<SkyKey> present = graph.getSuccessfulValues(map.values()).keySet();
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000408 if (present.size() == targets.size()) {
409 // Optimize for case of all targets being in graph.
410 return targets;
411 }
412 return Maps.filterValues(map, Predicates.in(present)).keySet();
413 }
414
Nathan Harmata3fae3662015-04-22 20:10:48 +0000415 @Override
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000416 protected Map<String, Set<Target>> preloadOrThrow(
417 QueryExpression caller, Collection<String> patterns)
418 throws QueryException, TargetParsingException {
Nathan Harmata3fae3662015-04-22 20:10:48 +0000419 GraphBackedRecursivePackageProvider provider =
Mark Schallerd7311e02015-07-07 16:36:09 +0000420 new GraphBackedRecursivePackageProvider(graph, universeTargetPatternKeys);
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000421 Map<String, Set<Target>> result = Maps.newHashMapWithExpectedSize(patterns.size());
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000422 for (String pattern : patterns) {
423 SkyKey patternKey = TargetPatternValue.key(pattern,
424 TargetPatternEvaluator.DEFAULT_FILTERING_POLICY, parserPrefix);
Mark Schallerb889cf32015-03-17 20:55:30 +0000425
Mark Schaller248f0442015-05-19 17:59:36 +0000426 TargetPatternValue.TargetPatternKey targetPatternKey =
427 ((TargetPatternValue.TargetPatternKey) patternKey.argument());
Mark Schallerb889cf32015-03-17 20:55:30 +0000428
Mark Schaller895b3d22015-03-23 14:27:26 +0000429 TargetParsingException targetParsingException = null;
Mark Schallerb889cf32015-03-17 20:55:30 +0000430 if (graph.exists(patternKey)) {
Nathan Harmata3fae3662015-04-22 20:10:48 +0000431 // The graph already contains a value or exception for this target pattern, so we use it.
Mark Schallerb889cf32015-03-17 20:55:30 +0000432 TargetPatternValue value = (TargetPatternValue) graph.getValue(patternKey);
433 if (value != null) {
Janak Ramakrishnan36a2e832015-08-25 14:26:34 +0000434 result.put(
435 pattern, ImmutableSet.copyOf(makeTargetsFromLabels(value.getTargets().getTargets())));
Mark Schallerb889cf32015-03-17 20:55:30 +0000436 } else {
Janak Ramakrishnanf7ff6162015-06-02 20:20:31 +0000437 // Because the graph was always initialized via a keep_going build, we know that the
438 // exception stored here must be a TargetParsingException. Thus the comment in
439 // SkyframeTargetPatternEvaluator#parseTargetPatternKeys describing the situation in which
440 // the exception acceptance must be looser does not apply here.
Mark Schaller895b3d22015-03-23 14:27:26 +0000441 targetParsingException =
442 (TargetParsingException)
443 Preconditions.checkNotNull(graph.getException(patternKey), pattern);
Mark Schallerb889cf32015-03-17 20:55:30 +0000444 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000445 } else {
Mark Schallerb889cf32015-03-17 20:55:30 +0000446 // If the graph doesn't contain a value for this target pattern, try to directly evaluate
447 // it, by making use of packages already present in the graph.
Mark Schallerb889cf32015-03-17 20:55:30 +0000448 RecursivePackageProviderBackedTargetPatternResolver resolver =
449 new RecursivePackageProviderBackedTargetPatternResolver(provider, eventHandler,
Mark Schaller248f0442015-05-19 17:59:36 +0000450 targetPatternKey.getPolicy(), pkgPath);
Mark Schaller66b35f32015-05-19 21:19:37 +0000451 TargetPattern parsedPattern = targetPatternKey.getParsedPattern();
Mark Schallerb889cf32015-03-17 20:55:30 +0000452 try {
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000453 result.put(pattern, filterTargetsNotInGraph(parsedPattern.eval(resolver).getTargets()));
Mark Schaller895b3d22015-03-23 14:27:26 +0000454 } catch (TargetParsingException e) {
455 targetParsingException = e;
Mark Schallerb889cf32015-03-17 20:55:30 +0000456 } catch (InterruptedException e) {
457 throw new QueryException(e.getMessage());
458 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000459 }
Mark Schaller895b3d22015-03-23 14:27:26 +0000460
461 if (targetParsingException != null) {
462 if (!keepGoing) {
463 throw targetParsingException;
464 } else {
465 eventHandler.handle(Event.error("Evaluation of query \"" + caller + "\" failed: "
466 + targetParsingException.getMessage()));
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000467 result.put(pattern, ImmutableSet.<Target>of());
Mark Schaller895b3d22015-03-23 14:27:26 +0000468 }
469 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000470 }
471 return result;
472 }
473
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000474 private static final Function<SkyKey, Label> SKYKEY_TO_LABEL = new Function<SkyKey, Label>() {
475 @Nullable
476 @Override
477 public Label apply(SkyKey skyKey) {
478 SkyFunctionName functionName = skyKey.functionName();
Mark Schaller8ff5b3c2015-07-29 17:32:11 +0000479 if (!functionName.equals(SkyFunctions.TRANSITIVE_TRAVERSAL)) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000480 // Skip non-targets.
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000481 return null;
482 }
483 return (Label) skyKey.argument();
484 }
485 };
486
487 private Map<SkyKey, Target> makeTargetsWithAssociations(Iterable<SkyKey> keys) {
488 return makeTargetsWithAssociations(keys, SKYKEY_TO_LABEL);
489 }
490
491 private Collection<Target> makeTargetsFromLabels(Iterable<Label> labels) {
492 return makeTargetsWithAssociations(labels, Functions.<Label>identity()).values();
493 }
494
495 private <E> Map<E, Target> makeTargetsWithAssociations(Iterable<E> keys,
496 Function<E, Label> toLabel) {
497 Multimap<SkyKey, E> packageKeyToTargetKeyMap = ArrayListMultimap.create();
498 for (E key : keys) {
499 Label label = toLabel.apply(key);
500 if (label == null) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000501 continue;
502 }
503 try {
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000504 packageKeyToTargetKeyMap.put(getPackageKeyAndValidateLabel(label), key);
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000505 } catch (QueryException e) {
506 // Skip disallowed labels.
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000507 }
508 }
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000509 ImmutableMap.Builder<E, Target> result = ImmutableMap.builder();
Miguel Alcon Pinto45820872015-09-11 19:57:47 +0000510 Map<SkyKey, SkyValue> packageMap = graph.getSuccessfulValues(packageKeyToTargetKeyMap.keySet());
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000511 for (Map.Entry<SkyKey, SkyValue> entry : packageMap.entrySet()) {
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000512 for (E targetKey : packageKeyToTargetKeyMap.get(entry.getKey())) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000513 try {
514 result.put(targetKey, ((PackageValue) entry.getValue()).getPackage()
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000515 .getTarget((toLabel.apply(targetKey)).getName()));
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000516 } catch (NoSuchTargetException e) {
517 // Skip missing target.
518 }
519 }
520 }
521 return result.build();
522 }
523
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000524 private static final Function<Target, SkyKey> TARGET_TO_SKY_KEY =
525 new Function<Target, SkyKey>() {
526 @Override
527 public SkyKey apply(Target target) {
528 return TransitiveTraversalValue.key(target.getLabel());
529 }
530 };
531
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000532 private static Iterable<SkyKey> makeKeys(Iterable<Target> targets) {
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000533 return Iterables.transform(targets, TARGET_TO_SKY_KEY);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000534 }
535
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000536 @Override
537 public Target getOrCreate(Target target) {
538 return target;
539 }
Janak Ramakrishnan643063d2015-06-25 16:21:49 +0000540
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000541 /**
Ulf Adams3ab82f72015-09-04 12:10:53 +0000542 * Get SkyKeys for the FileValues for the given {@code pathFragments}. To do this, we look for a
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000543 * package lookup node for each path fragment, since package lookup nodes contain the "root" of a
544 * package. The returned SkyKeys correspond to FileValues that may not exist in the graph.
545 */
546 private Collection<SkyKey> getSkyKeysForFileFragments(Iterable<PathFragment> pathFragments) {
547 Set<SkyKey> result = new HashSet<>();
548 Multimap<PathFragment, PathFragment> currentToOriginal = ArrayListMultimap.create();
549 for (PathFragment pathFragment : pathFragments) {
550 currentToOriginal.put(pathFragment, pathFragment);
551 }
552 while (!currentToOriginal.isEmpty()) {
553 Map<SkyKey, PathFragment> keys = new HashMap<>();
554 for (PathFragment pathFragment : currentToOriginal.keySet()) {
555 keys.put(
556 PackageLookupValue.key(PackageIdentifier.createInDefaultRepo(pathFragment)),
557 pathFragment);
558 }
Miguel Alcon Pinto45820872015-09-11 19:57:47 +0000559 Map<SkyKey, SkyValue> lookupValues = graph.getSuccessfulValues(keys.keySet());
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000560 for (Map.Entry<SkyKey, SkyValue> entry : lookupValues.entrySet()) {
561 PackageLookupValue packageLookupValue = (PackageLookupValue) entry.getValue();
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000562 if (packageLookupValue.packageExists()) {
John Field90ae6b92015-09-09 15:11:24 +0000563 PathFragment dir = keys.get(entry.getKey());
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000564 Collection<PathFragment> originalFiles = currentToOriginal.get(dir);
565 Preconditions.checkState(!originalFiles.isEmpty(), entry);
566 for (PathFragment fileName : originalFiles) {
567 result.add(
568 FileValue.key(RootedPath.toRootedPath(packageLookupValue.getRoot(), fileName)));
569 }
John Field90ae6b92015-09-09 15:11:24 +0000570 currentToOriginal.removeAll(dir);
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000571 }
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000572 }
573 Multimap<PathFragment, PathFragment> newCurrentToOriginal = ArrayListMultimap.create();
574 for (PathFragment pathFragment : currentToOriginal.keySet()) {
575 PathFragment parent = pathFragment.getParentDirectory();
576 if (parent != null) {
577 newCurrentToOriginal.putAll(parent, currentToOriginal.get(pathFragment));
578 }
579 }
580 currentToOriginal = newCurrentToOriginal;
581 }
582 return result;
583 }
584
585 /**
586 * Calculates the set of {@link Package} objects, represented as source file targets, that depend
587 * on the given list of BUILD files and subincludes (other files are filtered out).
588 */
589 @Nullable
590 Set<Target> getRBuildFiles(Collection<PathFragment> fileIdentifiers) {
591 Collection<SkyKey> files = getSkyKeysForFileFragments(fileIdentifiers);
Miguel Alcon Pinto45820872015-09-11 19:57:47 +0000592 Collection<SkyKey> current = graph.getSuccessfulValues(files).keySet();
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000593 Set<SkyKey> resultKeys = CompactHashSet.create();
594 while (!current.isEmpty()) {
595 Collection<Iterable<SkyKey>> reverseDeps = graph.getReverseDeps(current).values();
596 current = new HashSet<>();
597 for (SkyKey rdep : Iterables.concat(reverseDeps)) {
598 if (rdep.functionName().equals(SkyFunctions.PACKAGE)) {
599 resultKeys.add(rdep);
600 } else if (!rdep.functionName().equals(SkyFunctions.PACKAGE_LOOKUP)) {
601 // Packages may depend on subpackages for existence, but we don't report them as rdeps.
602 current.add(rdep);
603 }
604 }
605 }
Miguel Alcon Pinto45820872015-09-11 19:57:47 +0000606 Map<SkyKey, SkyValue> packageValues = graph.getSuccessfulValues(resultKeys);
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000607 ImmutableSet.Builder<Target> result = ImmutableSet.builder();
608 for (SkyValue value : packageValues.values()) {
Janak Ramakrishnan0a4c6e42015-09-17 00:37:58 +0000609 Package pkg = ((PackageValue) value).getPackage();
610 if (!pkg.containsErrors()) {
611 result.add(pkg.getBuildFile());
612 }
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000613 }
614 return result.build();
615 }
616
Janak Ramakrishnan643063d2015-06-25 16:21:49 +0000617 @Override
618 public Iterable<QueryFunction> getFunctions() {
619 return ImmutableList.<QueryFunction>builder()
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000620 .addAll(super.getFunctions())
621 .add(new AllRdepsFunction())
622 .add(new RBuildFilesFunction())
623 .build();
Janak Ramakrishnan643063d2015-06-25 16:21:49 +0000624 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000625}