blob: f1485e556a8e2262259d062b9719e7161b9c8614 [file] [log] [blame]
Janak Ramakrishnane72d5222015-02-26 17:09:18 +00001// Copyright 2015 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.query2;
15
Janak Ramakrishnan36858732015-06-17 16:45:47 +000016import com.google.common.base.Function;
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +000017import com.google.common.base.Functions;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000018import com.google.common.base.Preconditions;
19import com.google.common.base.Predicate;
Janak Ramakrishnanb735d2f2015-06-16 17:46:36 +000020import com.google.common.base.Predicates;
Janak Ramakrishnanf6f0fcc2015-06-19 20:24:52 +000021import com.google.common.collect.ArrayListMultimap;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000022import com.google.common.collect.Collections2;
Janak Ramakrishnancda5b662015-06-18 23:46:36 +000023import com.google.common.collect.ImmutableList;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000024import com.google.common.collect.ImmutableMap;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000025import com.google.common.collect.ImmutableSet;
26import com.google.common.collect.Iterables;
27import com.google.common.collect.Maps;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000028import com.google.common.collect.Multimap;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000029import com.google.common.collect.Sets;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000030import com.google.devtools.build.lib.cmdline.ResolvedTargets;
31import com.google.devtools.build.lib.cmdline.TargetParsingException;
Mark Schallerb889cf32015-03-17 20:55:30 +000032import com.google.devtools.build.lib.cmdline.TargetPattern;
David Chend6594262015-08-21 09:39:07 +000033import com.google.devtools.build.lib.collect.CompactHashSet;
Mark Schaller895b3d22015-03-23 14:27:26 +000034import com.google.devtools.build.lib.events.Event;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000035import com.google.devtools.build.lib.events.EventHandler;
36import com.google.devtools.build.lib.graph.Digraph;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000037import com.google.devtools.build.lib.packages.NoSuchTargetException;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000038import com.google.devtools.build.lib.packages.NoSuchThingException;
39import com.google.devtools.build.lib.packages.Package;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000040import com.google.devtools.build.lib.packages.PackageIdentifier;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000041import com.google.devtools.build.lib.packages.Rule;
42import com.google.devtools.build.lib.packages.Target;
Mark Schallerb889cf32015-03-17 20:55:30 +000043import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000044import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
Eric Fellheimera39f8a92015-07-28 19:11:23 +000045import com.google.devtools.build.lib.profiler.Profiler;
Janak Ramakrishnan643063d2015-06-25 16:21:49 +000046import com.google.devtools.build.lib.query2.engine.AllRdepsFunction;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000047import com.google.devtools.build.lib.query2.engine.QueryEvalResult;
48import com.google.devtools.build.lib.query2.engine.QueryException;
49import com.google.devtools.build.lib.query2.engine.QueryExpression;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000050import com.google.devtools.build.lib.skyframe.FileValue;
Mark Schallerb889cf32015-03-17 20:55:30 +000051import com.google.devtools.build.lib.skyframe.GraphBackedRecursivePackageProvider;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000052import com.google.devtools.build.lib.skyframe.PackageLookupValue;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000053import com.google.devtools.build.lib.skyframe.PackageValue;
Mark Schallerd7311e02015-07-07 16:36:09 +000054import com.google.devtools.build.lib.skyframe.PrepareDepsOfPatternsValue;
Mark Schallerb889cf32015-03-17 20:55:30 +000055import com.google.devtools.build.lib.skyframe.RecursivePackageProviderBackedTargetPatternResolver;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000056import com.google.devtools.build.lib.skyframe.SkyFunctions;
57import com.google.devtools.build.lib.skyframe.TargetPatternValue;
Mark Schallerd7311e02015-07-07 16:36:09 +000058import com.google.devtools.build.lib.skyframe.TargetPatternValue.TargetPatternKey;
Mark Schaller8ff5b3c2015-07-29 17:32:11 +000059import com.google.devtools.build.lib.skyframe.TransitiveTraversalValue;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000060import com.google.devtools.build.lib.syntax.Label;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000061import com.google.devtools.build.lib.vfs.PathFragment;
62import com.google.devtools.build.lib.vfs.RootedPath;
Mark Schallerd7311e02015-07-07 16:36:09 +000063import com.google.devtools.build.skyframe.EvaluationResult;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000064import com.google.devtools.build.skyframe.SkyFunctionName;
65import com.google.devtools.build.skyframe.SkyKey;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000066import com.google.devtools.build.skyframe.SkyValue;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000067import com.google.devtools.build.skyframe.WalkableGraph;
68import com.google.devtools.build.skyframe.WalkableGraph.WalkableGraphFactory;
69
70import java.util.ArrayDeque;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000071import java.util.Collection;
72import java.util.Deque;
73import java.util.HashMap;
74import java.util.HashSet;
75import java.util.Iterator;
76import java.util.LinkedHashSet;
77import java.util.List;
78import java.util.Map;
79import java.util.Set;
Eric Fellheimera39f8a92015-07-28 19:11:23 +000080import java.util.logging.Logger;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000081
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +000082import javax.annotation.Nullable;
83
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000084/**
85 * {@link AbstractBlazeQueryEnvironment} that introspects the Skyframe graph to find forward and
86 * reverse edges. Results obtained by calling {@link #evaluateQuery} are not guaranteed to be in
87 * any particular order. As well, this class eagerly loads the full transitive closure of targets,
88 * even if the full closure isn't needed.
89 */
90public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> {
91 private WalkableGraph graph;
92
Mark Schallerd7311e02015-07-07 16:36:09 +000093 private ImmutableList<TargetPatternKey> universeTargetPatternKeys;
94
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000095 private final BlazeTargetAccessor accessor = new BlazeTargetAccessor(this);
96 private final int loadingPhaseThreads;
97 private final WalkableGraphFactory graphFactory;
98 private final List<String> universeScope;
99 private final String parserPrefix;
Mark Schallerb889cf32015-03-17 20:55:30 +0000100 private final PathPackageLocator pkgPath;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000101
Eric Fellheimera39f8a92015-07-28 19:11:23 +0000102 private static final Logger LOG = Logger.getLogger(SkyQueryEnvironment.class.getName());
103
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000104 public SkyQueryEnvironment(boolean keepGoing, boolean strictScope, int loadingPhaseThreads,
105 Predicate<Label> labelFilter,
106 EventHandler eventHandler,
107 Set<Setting> settings,
108 Iterable<QueryFunction> extraFunctions, String parserPrefix,
109 WalkableGraphFactory graphFactory,
Mark Schallerb889cf32015-03-17 20:55:30 +0000110 List<String> universeScope, PathPackageLocator pkgPath) {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000111 super(keepGoing, strictScope, labelFilter,
112 eventHandler,
113 settings,
114 extraFunctions);
115 this.loadingPhaseThreads = loadingPhaseThreads;
116 this.graphFactory = graphFactory;
Mark Schallerb889cf32015-03-17 20:55:30 +0000117 this.pkgPath = pkgPath;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000118 this.universeScope = Preconditions.checkNotNull(universeScope);
119 this.parserPrefix = parserPrefix;
120 Preconditions.checkState(!universeScope.isEmpty(),
121 "No queries can be performed with an empty universe");
122 }
123
124 private void init() throws InterruptedException {
Eric Fellheimera39f8a92015-07-28 19:11:23 +0000125 long startTime = Profiler.nanoTimeMaybe();
Mark Schallerd7311e02015-07-07 16:36:09 +0000126 EvaluationResult<SkyValue> result =
127 graphFactory.prepareAndGet(universeScope, loadingPhaseThreads, eventHandler);
128 graph = result.getWalkableGraph();
129 Collection<SkyValue> values = result.values();
Eric Fellheimera39f8a92015-07-28 19:11:23 +0000130 long duration = Profiler.nanoTimeMaybe() - startTime;
131 if (duration > 0) {
132 LOG.info("Spent " + (duration / 1000 / 1000) + " ms on evaluation and walkable graph");
133 }
Mark Schallerd7311e02015-07-07 16:36:09 +0000134
135 // The universe query may fail if there are errors during its evaluation, e.g. because of
136 // cycles in the target graph.
137 boolean singleValueEvaluated = values.size() == 1;
138 boolean foundError = !result.errorMap().isEmpty();
139 boolean evaluationFoundCycle =
140 foundError && !Iterables.isEmpty(result.getError().getCycleInfo());
141 Preconditions.checkState(singleValueEvaluated || evaluationFoundCycle,
142 "Universe query \"%s\" unexpectedly did not result in a single value as expected (%s"
143 + " values in result) and it did not fail because of a cycle.%s",
144 universeScope, values.size(), foundError ? " Error: " + result.getError().toString() : "");
145 if (singleValueEvaluated) {
146 PrepareDepsOfPatternsValue prepareDepsOfPatternsValue =
147 (PrepareDepsOfPatternsValue) Iterables.getOnlyElement(values);
148 universeTargetPatternKeys = prepareDepsOfPatternsValue.getTargetPatternKeys();
149 } else {
150 // The error is because of a cycle, so keep going with the graph we managed to load.
151 universeTargetPatternKeys = ImmutableList.of();
152 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000153 }
154
155 @Override
156 public QueryEvalResult<Target> evaluateQuery(QueryExpression expr)
Janak Ramakrishnan89907392015-07-08 21:43:31 +0000157 throws QueryException, InterruptedException {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000158 // Some errors are reported as QueryExceptions and others as ERROR events (if --keep_going). The
159 // result is set to have an error iff there were errors emitted during the query, so we reset
160 // errors here.
161 eventHandler.resetErrors();
Janak Ramakrishnan89907392015-07-08 21:43:31 +0000162 init();
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000163 return super.evaluateQuery(expr);
164 }
165
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000166 private Map<Target, Collection<Target>> makeTargetsMap(Map<SkyKey, Iterable<SkyKey>> input) {
167 ImmutableMap.Builder<Target, Collection<Target>> result = ImmutableMap.builder();
168
169 for (Map.Entry<SkyKey, Target> entry : makeTargetsWithAssociations(input.keySet()).entrySet()) {
170 result.put(entry.getValue(), makeTargets(input.get(entry.getKey())));
171 }
172 return result.build();
173 }
174
175 private Map<Target, Collection<Target>> getRawFwdDeps(Iterable<Target> targets) {
176 return makeTargetsMap(graph.getDirectDeps(makeKeys(targets)));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000177 }
178
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000179 private Map<Target, Collection<Target>> getRawReverseDeps(Iterable<Target> targets) {
180 return makeTargetsMap(graph.getReverseDeps(makeKeys(targets)));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000181 }
182
183 private Set<Label> getAllowedDeps(Rule rule) {
Miguel Alcon Pinto4ffe28d2015-08-19 14:29:02 +0000184 Set<Label> allowedLabels = new HashSet<>(rule.getTransitions(dependencyFilter).values());
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000185 allowedLabels.addAll(rule.getVisibility().getDependencyLabels());
Marian Loburfdd788e2015-03-25 09:36:28 +0000186 // We should add deps from aspects, otherwise they are going to be filtered out.
187 allowedLabels.addAll(rule.getAspectLabelsSuperset(dependencyFilter));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000188 return allowedLabels;
189 }
190
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000191 private Collection<Target> filterFwdDeps(Target target, Collection<Target> rawFwdDeps) {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000192 if (!(target instanceof Rule)) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000193 return rawFwdDeps;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000194 }
195 final Set<Label> allowedLabels = getAllowedDeps((Rule) target);
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000196 return Collections2.filter(rawFwdDeps,
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000197 new Predicate<Target>() {
198 @Override
199 public boolean apply(Target target) {
200 return allowedLabels.contains(target.getLabel());
201 }
202 });
203 }
204
Nathan Harmata3fae3662015-04-22 20:10:48 +0000205 @Override
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000206 public Collection<Target> getFwdDeps(Iterable<Target> targets) {
207 Set<Target> result = new HashSet<>();
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000208 for (Map.Entry<Target, Collection<Target>> entry : getRawFwdDeps(targets).entrySet()) {
209 result.addAll(filterFwdDeps(entry.getKey(), entry.getValue()));
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000210 }
211 return result;
212 }
213
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000214 private Collection<Target> filterReverseDeps(final Target target,
215 Collection<Target> rawReverseDeps) {
216 return Collections2.filter(rawReverseDeps, new Predicate<Target>() {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000217 @Override
218 public boolean apply(Target parent) {
219 return !(parent instanceof Rule)
220 || getAllowedDeps((Rule) parent).contains(target.getLabel());
221 }
222 });
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000223
224 }
225
226 @Override
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000227 public Collection<Target> getReverseDeps(Iterable<Target> targets) {
228 Set<Target> result = new HashSet<>();
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000229 for (Map.Entry<Target, Collection<Target>> entry : getRawReverseDeps(targets).entrySet()) {
230 result.addAll(filterReverseDeps(entry.getKey(), entry.getValue()));
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000231 }
232 return result;
233 }
234
235 @Override
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000236 public Set<Target> getTransitiveClosure(Set<Target> targets) {
237 Set<Target> visited = new HashSet<>();
Janak Ramakrishnanb735d2f2015-06-16 17:46:36 +0000238 Collection<Target> current = targets;
239 while (!current.isEmpty()) {
240 Collection<Target> toVisit = Collections2.filter(current,
241 Predicates.not(Predicates.in(visited)));
242 current = getFwdDeps(toVisit);
243 visited.addAll(toVisit);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000244 }
Janak Ramakrishnanb735d2f2015-06-16 17:46:36 +0000245 return ImmutableSet.copyOf(visited);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000246 }
247
248 // Implemented with a breadth-first search.
249 @Override
250 public Set<Target> getNodesOnPath(Target from, Target to) {
251 // Tree of nodes visited so far.
252 Map<Target, Target> nodeToParent = new HashMap<>();
253 // Contains all nodes left to visit in a (LIFO) stack.
254 Deque<Target> toVisit = new ArrayDeque<>();
255 toVisit.add(from);
256 nodeToParent.put(from, null);
257 while (!toVisit.isEmpty()) {
258 Target current = toVisit.removeFirst();
259 if (to.equals(current)) {
260 return ImmutableSet.copyOf(Digraph.getPathToTreeNode(nodeToParent, to));
261 }
Janak Ramakrishnancda5b662015-06-18 23:46:36 +0000262 for (Target dep : getFwdDeps(ImmutableList.of(current))) {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000263 if (!nodeToParent.containsKey(dep)) {
264 nodeToParent.put(dep, current);
265 toVisit.addFirst(dep);
266 }
267 }
268 }
269 // Note that the only current caller of this method checks first to see if there is a path
270 // before calling this method. It is not clear what the return value should be here.
271 return null;
272 }
273
274 @Override
275 public Set<Target> getTargetsMatchingPattern(QueryExpression owner, String pattern)
276 throws QueryException {
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000277 Set<Target> targets = new LinkedHashSet<>(resolvedTargetPatterns.get(pattern));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000278
279 // Sets.filter would be more convenient here, but can't deal with exceptions.
280 Iterator<Target> targetIterator = targets.iterator();
281 while (targetIterator.hasNext()) {
282 Target target = targetIterator.next();
283 if (!validateScope(target.getLabel(), strictScope)) {
284 targetIterator.remove();
285 }
286 }
287 return targets;
288 }
289
290 @Override
291 public Set<Target> getBuildFiles(QueryExpression caller, Set<Target> nodes)
292 throws QueryException {
293 Set<Target> dependentFiles = new LinkedHashSet<>();
294 Set<Package> seenPackages = new HashSet<>();
295 // Keep track of seen labels, to avoid adding a fake subinclude label that also exists as a
296 // real target.
297 Set<Label> seenLabels = new HashSet<>();
298
299 // Adds all the package definition files (BUILD files and build
300 // extensions) for package "pkg", to "buildfiles".
301 for (Target x : nodes) {
302 Package pkg = x.getPackage();
303 if (seenPackages.add(pkg)) {
304 addIfUniqueLabel(pkg.getBuildFile(), seenLabels, dependentFiles);
305 for (Label subinclude
306 : Iterables.concat(pkg.getSubincludeLabels(), pkg.getSkylarkFileDependencies())) {
307 addIfUniqueLabel(getSubincludeTarget(subinclude, pkg), seenLabels, dependentFiles);
308
309 // Also add the BUILD file of the subinclude.
310 try {
311 addIfUniqueLabel(getSubincludeTarget(
312 subinclude.getLocalTargetLabel("BUILD"), pkg), seenLabels, dependentFiles);
313 } catch (Label.SyntaxException e) {
314 throw new AssertionError("BUILD should always parse as a target name", e);
315 }
316 }
317 }
318 }
319 return dependentFiles;
320 }
321
322 private static void addIfUniqueLabel(Target node, Set<Label> labels, Set<Target> nodes) {
323 if (labels.add(node.getLabel())) {
324 nodes.add(node);
325 }
326 }
327
328 private static Target getSubincludeTarget(final Label label, Package pkg) {
Mark Schallerb817e3f2015-06-04 17:14:18 +0000329 return new FakeSubincludeTarget(label, pkg);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000330 }
331
332 @Override
333 public TargetAccessor<Target> getAccessor() {
334 return accessor;
335 }
336
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000337 private SkyKey getPackageKeyAndValidateLabel(Label label) throws QueryException {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000338 // Can't use strictScope here because we are expecting a target back.
339 validateScope(label, true);
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000340 return PackageValue.key(label.getPackageIdentifier());
341 }
342
343 @Override
344 public Target getTarget(Label label) throws TargetNotFoundException, QueryException {
345 SkyKey packageKey = getPackageKeyAndValidateLabel(label);
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000346 if (!graph.exists(packageKey)) {
347 throw new QueryException(packageKey + " does not exist in graph");
348 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000349 try {
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000350 PackageValue packageValue = (PackageValue) graph.getValue(packageKey);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000351 if (packageValue != null) {
352 return packageValue.getPackage().getTarget(label.getName());
353 } else {
354 throw (NoSuchThingException) Preconditions.checkNotNull(
355 graph.getException(packageKey), label);
356 }
357 } catch (NoSuchThingException e) {
358 throw new TargetNotFoundException(e);
359 }
360 }
361
362 @Override
363 public void buildTransitiveClosure(QueryExpression caller, Set<Target> targets, int maxDepth)
364 throws QueryException {
365 // Everything has already been loaded, so here we just check for errors so that we can
366 // pre-emptively throw/report if needed.
Janak Ramakrishnanf6f0fcc2015-06-19 20:24:52 +0000367 for (Map.Entry<SkyKey, Exception> entry :
368 graph.getMissingAndExceptions(makeKeys(targets)).entrySet()) {
369 if (entry.getValue() == null) {
370 throw new QueryException(entry.getKey().argument() + " does not exist in graph");
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000371 }
Janak Ramakrishnanf6f0fcc2015-06-19 20:24:52 +0000372 reportBuildFileError(caller, entry.getValue().getMessage());
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000373 }
374 }
375
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000376 private Set<Target> filterTargetsNotInGraph(Set<Target> targets) {
377 Map<Target, SkyKey> map = Maps.toMap(targets, TARGET_TO_SKY_KEY);
378 Set<SkyKey> present = graph.getDoneValues(map.values()).keySet();
379 if (present.size() == targets.size()) {
380 // Optimize for case of all targets being in graph.
381 return targets;
382 }
383 return Maps.filterValues(map, Predicates.in(present)).keySet();
384 }
385
Nathan Harmata3fae3662015-04-22 20:10:48 +0000386 @Override
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000387 protected Map<String, Set<Target>> preloadOrThrow(
388 QueryExpression caller, Collection<String> patterns)
389 throws QueryException, TargetParsingException {
Nathan Harmata3fae3662015-04-22 20:10:48 +0000390 GraphBackedRecursivePackageProvider provider =
Mark Schallerd7311e02015-07-07 16:36:09 +0000391 new GraphBackedRecursivePackageProvider(graph, universeTargetPatternKeys);
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000392 Map<String, Set<Target>> result = Maps.newHashMapWithExpectedSize(patterns.size());
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000393 for (String pattern : patterns) {
394 SkyKey patternKey = TargetPatternValue.key(pattern,
395 TargetPatternEvaluator.DEFAULT_FILTERING_POLICY, parserPrefix);
Mark Schallerb889cf32015-03-17 20:55:30 +0000396
Mark Schaller248f0442015-05-19 17:59:36 +0000397 TargetPatternValue.TargetPatternKey targetPatternKey =
398 ((TargetPatternValue.TargetPatternKey) patternKey.argument());
Mark Schallerb889cf32015-03-17 20:55:30 +0000399
Mark Schaller895b3d22015-03-23 14:27:26 +0000400 TargetParsingException targetParsingException = null;
Mark Schallerb889cf32015-03-17 20:55:30 +0000401 if (graph.exists(patternKey)) {
Nathan Harmata3fae3662015-04-22 20:10:48 +0000402 // The graph already contains a value or exception for this target pattern, so we use it.
Mark Schallerb889cf32015-03-17 20:55:30 +0000403 TargetPatternValue value = (TargetPatternValue) graph.getValue(patternKey);
404 if (value != null) {
Nathan Harmata3fae3662015-04-22 20:10:48 +0000405 ResolvedTargets.Builder<Target> targetsBuilder = ResolvedTargets.builder();
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000406 targetsBuilder.addAll(makeTargetsFromLabels(value.getTargets().getTargets()));
407 targetsBuilder.removeAll(makeTargetsFromLabels(value.getTargets().getFilteredTargets()));
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000408 result.put(pattern, targetsBuilder.build().getTargets());
Mark Schallerb889cf32015-03-17 20:55:30 +0000409 } else {
Janak Ramakrishnanf7ff6162015-06-02 20:20:31 +0000410 // Because the graph was always initialized via a keep_going build, we know that the
411 // exception stored here must be a TargetParsingException. Thus the comment in
412 // SkyframeTargetPatternEvaluator#parseTargetPatternKeys describing the situation in which
413 // the exception acceptance must be looser does not apply here.
Mark Schaller895b3d22015-03-23 14:27:26 +0000414 targetParsingException =
415 (TargetParsingException)
416 Preconditions.checkNotNull(graph.getException(patternKey), pattern);
Mark Schallerb889cf32015-03-17 20:55:30 +0000417 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000418 } else {
Mark Schallerb889cf32015-03-17 20:55:30 +0000419 // If the graph doesn't contain a value for this target pattern, try to directly evaluate
420 // it, by making use of packages already present in the graph.
Mark Schallerb889cf32015-03-17 20:55:30 +0000421 RecursivePackageProviderBackedTargetPatternResolver resolver =
422 new RecursivePackageProviderBackedTargetPatternResolver(provider, eventHandler,
Mark Schaller248f0442015-05-19 17:59:36 +0000423 targetPatternKey.getPolicy(), pkgPath);
Mark Schaller66b35f32015-05-19 21:19:37 +0000424 TargetPattern parsedPattern = targetPatternKey.getParsedPattern();
Mark Schallerb889cf32015-03-17 20:55:30 +0000425 try {
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000426 result.put(pattern, filterTargetsNotInGraph(parsedPattern.eval(resolver).getTargets()));
Mark Schaller895b3d22015-03-23 14:27:26 +0000427 } catch (TargetParsingException e) {
428 targetParsingException = e;
Mark Schallerb889cf32015-03-17 20:55:30 +0000429 } catch (InterruptedException e) {
430 throw new QueryException(e.getMessage());
431 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000432 }
Mark Schaller895b3d22015-03-23 14:27:26 +0000433
434 if (targetParsingException != null) {
435 if (!keepGoing) {
436 throw targetParsingException;
437 } else {
438 eventHandler.handle(Event.error("Evaluation of query \"" + caller + "\" failed: "
439 + targetParsingException.getMessage()));
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000440 result.put(pattern, ImmutableSet.<Target>of());
Mark Schaller895b3d22015-03-23 14:27:26 +0000441 }
442 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000443 }
444 return result;
445 }
446
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000447 private Collection<Target> makeTargets(Iterable<SkyKey> keys) {
448 return makeTargetsWithAssociations(keys).values();
449 }
450
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000451 private static final Function<SkyKey, Label> SKYKEY_TO_LABEL = new Function<SkyKey, Label>() {
452 @Nullable
453 @Override
454 public Label apply(SkyKey skyKey) {
455 SkyFunctionName functionName = skyKey.functionName();
Mark Schaller8ff5b3c2015-07-29 17:32:11 +0000456 if (!functionName.equals(SkyFunctions.TRANSITIVE_TRAVERSAL)) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000457 // Skip non-targets.
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000458 return null;
459 }
460 return (Label) skyKey.argument();
461 }
462 };
463
464 private Map<SkyKey, Target> makeTargetsWithAssociations(Iterable<SkyKey> keys) {
465 return makeTargetsWithAssociations(keys, SKYKEY_TO_LABEL);
466 }
467
468 private Collection<Target> makeTargetsFromLabels(Iterable<Label> labels) {
469 return makeTargetsWithAssociations(labels, Functions.<Label>identity()).values();
470 }
471
472 private <E> Map<E, Target> makeTargetsWithAssociations(Iterable<E> keys,
473 Function<E, Label> toLabel) {
474 Multimap<SkyKey, E> packageKeyToTargetKeyMap = ArrayListMultimap.create();
475 for (E key : keys) {
476 Label label = toLabel.apply(key);
477 if (label == null) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000478 continue;
479 }
480 try {
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000481 packageKeyToTargetKeyMap.put(getPackageKeyAndValidateLabel(label), key);
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000482 } catch (QueryException e) {
483 // Skip disallowed labels.
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000484 }
485 }
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000486 ImmutableMap.Builder<E, Target> result = ImmutableMap.builder();
Janak Ramakrishnanf6f0fcc2015-06-19 20:24:52 +0000487 Map<SkyKey, SkyValue> packageMap = graph.getDoneValues(packageKeyToTargetKeyMap.keySet());
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000488 for (Map.Entry<SkyKey, SkyValue> entry : packageMap.entrySet()) {
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000489 for (E targetKey : packageKeyToTargetKeyMap.get(entry.getKey())) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000490 try {
491 result.put(targetKey, ((PackageValue) entry.getValue()).getPackage()
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000492 .getTarget((toLabel.apply(targetKey)).getName()));
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000493 } catch (NoSuchTargetException e) {
494 // Skip missing target.
495 }
496 }
497 }
498 return result.build();
499 }
500
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000501 private static final Function<Target, SkyKey> TARGET_TO_SKY_KEY =
502 new Function<Target, SkyKey>() {
503 @Override
504 public SkyKey apply(Target target) {
505 return TransitiveTraversalValue.key(target.getLabel());
506 }
507 };
508
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000509 private static Iterable<SkyKey> makeKeys(Iterable<Target> targets) {
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000510 return Iterables.transform(targets, TARGET_TO_SKY_KEY);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000511 }
512
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000513 @Override
514 public Target getOrCreate(Target target) {
515 return target;
516 }
Janak Ramakrishnan643063d2015-06-25 16:21:49 +0000517
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000518 /**
519 * Get SkyKeys for the FileValues for the given {@param pathFragments}. To do this, we look for a
520 * package lookup node for each path fragment, since package lookup nodes contain the "root" of a
521 * package. The returned SkyKeys correspond to FileValues that may not exist in the graph.
522 */
523 private Collection<SkyKey> getSkyKeysForFileFragments(Iterable<PathFragment> pathFragments) {
524 Set<SkyKey> result = new HashSet<>();
525 Multimap<PathFragment, PathFragment> currentToOriginal = ArrayListMultimap.create();
526 for (PathFragment pathFragment : pathFragments) {
527 currentToOriginal.put(pathFragment, pathFragment);
528 }
529 while (!currentToOriginal.isEmpty()) {
530 Map<SkyKey, PathFragment> keys = new HashMap<>();
531 for (PathFragment pathFragment : currentToOriginal.keySet()) {
532 keys.put(
533 PackageLookupValue.key(PackageIdentifier.createInDefaultRepo(pathFragment)),
534 pathFragment);
535 }
536 Map<SkyKey, SkyValue> lookupValues = graph.getDoneValues(keys.keySet());
537 for (Map.Entry<SkyKey, SkyValue> entry : lookupValues.entrySet()) {
538 PackageLookupValue packageLookupValue = (PackageLookupValue) entry.getValue();
539 PathFragment dir = keys.get(entry.getKey());
540 if (packageLookupValue.packageExists()) {
541 Collection<PathFragment> originalFiles = currentToOriginal.get(dir);
542 Preconditions.checkState(!originalFiles.isEmpty(), entry);
543 for (PathFragment fileName : originalFiles) {
544 result.add(
545 FileValue.key(RootedPath.toRootedPath(packageLookupValue.getRoot(), fileName)));
546 }
547 }
548 currentToOriginal.removeAll(dir);
549 }
550 Multimap<PathFragment, PathFragment> newCurrentToOriginal = ArrayListMultimap.create();
551 for (PathFragment pathFragment : currentToOriginal.keySet()) {
552 PathFragment parent = pathFragment.getParentDirectory();
553 if (parent != null) {
554 newCurrentToOriginal.putAll(parent, currentToOriginal.get(pathFragment));
555 }
556 }
557 currentToOriginal = newCurrentToOriginal;
558 }
559 return result;
560 }
561
562 /**
563 * Calculates the set of {@link Package} objects, represented as source file targets, that depend
564 * on the given list of BUILD files and subincludes (other files are filtered out).
565 */
566 @Nullable
567 Set<Target> getRBuildFiles(Collection<PathFragment> fileIdentifiers) {
568 Collection<SkyKey> files = getSkyKeysForFileFragments(fileIdentifiers);
569 Collection<SkyKey> current = graph.getDoneValues(files).keySet();
570 Set<SkyKey> resultKeys = CompactHashSet.create();
571 while (!current.isEmpty()) {
572 Collection<Iterable<SkyKey>> reverseDeps = graph.getReverseDeps(current).values();
573 current = new HashSet<>();
574 for (SkyKey rdep : Iterables.concat(reverseDeps)) {
575 if (rdep.functionName().equals(SkyFunctions.PACKAGE)) {
576 resultKeys.add(rdep);
577 } else if (!rdep.functionName().equals(SkyFunctions.PACKAGE_LOOKUP)) {
578 // Packages may depend on subpackages for existence, but we don't report them as rdeps.
579 current.add(rdep);
580 }
581 }
582 }
583 Map<SkyKey, SkyValue> packageValues = graph.getDoneValues(resultKeys);
584 if (packageValues.size() != resultKeys.size()) {
585 throw new IllegalStateException(
586 "Missing values: " + Sets.difference(resultKeys, packageValues.keySet()));
587 }
588 ImmutableSet.Builder<Target> result = ImmutableSet.builder();
589 for (SkyValue value : packageValues.values()) {
590 result.add(((PackageValue) value).getPackage().getBuildFile());
591 }
592 return result.build();
593 }
594
Janak Ramakrishnan643063d2015-06-25 16:21:49 +0000595 @Override
596 public Iterable<QueryFunction> getFunctions() {
597 return ImmutableList.<QueryFunction>builder()
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000598 .addAll(super.getFunctions())
599 .add(new AllRdepsFunction())
600 .add(new RBuildFilesFunction())
601 .build();
Janak Ramakrishnan643063d2015-06-25 16:21:49 +0000602 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000603}