blob: 0957b8192a607291e5bea43af587a7883d2586bb [file] [log] [blame]
Janak Ramakrishnane72d5222015-02-26 17:09:18 +00001// Copyright 2015 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package com.google.devtools.build.lib.query2;
15
Janak Ramakrishnan36858732015-06-17 16:45:47 +000016import com.google.common.base.Function;
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +000017import com.google.common.base.Functions;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000018import com.google.common.base.Preconditions;
19import com.google.common.base.Predicate;
Janak Ramakrishnanb735d2f2015-06-16 17:46:36 +000020import com.google.common.base.Predicates;
Janak Ramakrishnanf6f0fcc2015-06-19 20:24:52 +000021import com.google.common.collect.ArrayListMultimap;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000022import com.google.common.collect.Collections2;
Janak Ramakrishnancda5b662015-06-18 23:46:36 +000023import com.google.common.collect.ImmutableList;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000024import com.google.common.collect.ImmutableMap;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000025import com.google.common.collect.ImmutableSet;
26import com.google.common.collect.Iterables;
27import com.google.common.collect.Maps;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000028import com.google.common.collect.Multimap;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000029import com.google.common.collect.Sets;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000030import com.google.devtools.build.lib.cmdline.TargetParsingException;
Mark Schallerb889cf32015-03-17 20:55:30 +000031import com.google.devtools.build.lib.cmdline.TargetPattern;
David Chend6594262015-08-21 09:39:07 +000032import com.google.devtools.build.lib.collect.CompactHashSet;
Mark Schaller895b3d22015-03-23 14:27:26 +000033import com.google.devtools.build.lib.events.Event;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000034import com.google.devtools.build.lib.events.EventHandler;
35import com.google.devtools.build.lib.graph.Digraph;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000036import com.google.devtools.build.lib.packages.NoSuchTargetException;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000037import com.google.devtools.build.lib.packages.NoSuchThingException;
38import com.google.devtools.build.lib.packages.Package;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000039import com.google.devtools.build.lib.packages.PackageIdentifier;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000040import com.google.devtools.build.lib.packages.Rule;
41import com.google.devtools.build.lib.packages.Target;
Mark Schallerb889cf32015-03-17 20:55:30 +000042import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000043import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
Eric Fellheimera39f8a92015-07-28 19:11:23 +000044import com.google.devtools.build.lib.profiler.Profiler;
Janak Ramakrishnan643063d2015-06-25 16:21:49 +000045import com.google.devtools.build.lib.query2.engine.AllRdepsFunction;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000046import com.google.devtools.build.lib.query2.engine.QueryEvalResult;
47import com.google.devtools.build.lib.query2.engine.QueryException;
48import com.google.devtools.build.lib.query2.engine.QueryExpression;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000049import com.google.devtools.build.lib.skyframe.FileValue;
Mark Schallerb889cf32015-03-17 20:55:30 +000050import com.google.devtools.build.lib.skyframe.GraphBackedRecursivePackageProvider;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000051import com.google.devtools.build.lib.skyframe.PackageLookupValue;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000052import com.google.devtools.build.lib.skyframe.PackageValue;
Mark Schallerd7311e02015-07-07 16:36:09 +000053import com.google.devtools.build.lib.skyframe.PrepareDepsOfPatternsValue;
Mark Schallerb889cf32015-03-17 20:55:30 +000054import com.google.devtools.build.lib.skyframe.RecursivePackageProviderBackedTargetPatternResolver;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000055import com.google.devtools.build.lib.skyframe.SkyFunctions;
56import com.google.devtools.build.lib.skyframe.TargetPatternValue;
Mark Schallerd7311e02015-07-07 16:36:09 +000057import com.google.devtools.build.lib.skyframe.TargetPatternValue.TargetPatternKey;
Mark Schaller8ff5b3c2015-07-29 17:32:11 +000058import com.google.devtools.build.lib.skyframe.TransitiveTraversalValue;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000059import com.google.devtools.build.lib.syntax.Label;
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +000060import com.google.devtools.build.lib.vfs.PathFragment;
61import com.google.devtools.build.lib.vfs.RootedPath;
Mark Schallerd7311e02015-07-07 16:36:09 +000062import com.google.devtools.build.skyframe.EvaluationResult;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000063import com.google.devtools.build.skyframe.SkyFunctionName;
64import com.google.devtools.build.skyframe.SkyKey;
Janak Ramakrishnan36858732015-06-17 16:45:47 +000065import com.google.devtools.build.skyframe.SkyValue;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000066import com.google.devtools.build.skyframe.WalkableGraph;
67import com.google.devtools.build.skyframe.WalkableGraph.WalkableGraphFactory;
68
69import java.util.ArrayDeque;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000070import java.util.Collection;
71import java.util.Deque;
72import java.util.HashMap;
73import java.util.HashSet;
74import java.util.Iterator;
75import java.util.LinkedHashSet;
76import java.util.List;
77import java.util.Map;
78import java.util.Set;
Eric Fellheimera39f8a92015-07-28 19:11:23 +000079import java.util.logging.Logger;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000080
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +000081import javax.annotation.Nullable;
82
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000083/**
84 * {@link AbstractBlazeQueryEnvironment} that introspects the Skyframe graph to find forward and
85 * reverse edges. Results obtained by calling {@link #evaluateQuery} are not guaranteed to be in
86 * any particular order. As well, this class eagerly loads the full transitive closure of targets,
87 * even if the full closure isn't needed.
88 */
89public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> {
90 private WalkableGraph graph;
91
Mark Schallerd7311e02015-07-07 16:36:09 +000092 private ImmutableList<TargetPatternKey> universeTargetPatternKeys;
93
Janak Ramakrishnane72d5222015-02-26 17:09:18 +000094 private final BlazeTargetAccessor accessor = new BlazeTargetAccessor(this);
95 private final int loadingPhaseThreads;
96 private final WalkableGraphFactory graphFactory;
97 private final List<String> universeScope;
98 private final String parserPrefix;
Mark Schallerb889cf32015-03-17 20:55:30 +000099 private final PathPackageLocator pkgPath;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000100
Eric Fellheimera39f8a92015-07-28 19:11:23 +0000101 private static final Logger LOG = Logger.getLogger(SkyQueryEnvironment.class.getName());
102
Miguel Alcon Pintob45e2622015-08-21 18:31:23 +0000103 private static final Function<Target, Label> TARGET_LABEL_FUNCTION =
104 new Function<Target, Label>() {
105
106 @Override
107 public Label apply(Target target) {
108 return target.getLabel();
109 }
110 };
111
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000112 public SkyQueryEnvironment(boolean keepGoing, boolean strictScope, int loadingPhaseThreads,
113 Predicate<Label> labelFilter,
114 EventHandler eventHandler,
115 Set<Setting> settings,
116 Iterable<QueryFunction> extraFunctions, String parserPrefix,
117 WalkableGraphFactory graphFactory,
Mark Schallerb889cf32015-03-17 20:55:30 +0000118 List<String> universeScope, PathPackageLocator pkgPath) {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000119 super(keepGoing, strictScope, labelFilter,
120 eventHandler,
121 settings,
122 extraFunctions);
123 this.loadingPhaseThreads = loadingPhaseThreads;
124 this.graphFactory = graphFactory;
Mark Schallerb889cf32015-03-17 20:55:30 +0000125 this.pkgPath = pkgPath;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000126 this.universeScope = Preconditions.checkNotNull(universeScope);
127 this.parserPrefix = parserPrefix;
128 Preconditions.checkState(!universeScope.isEmpty(),
129 "No queries can be performed with an empty universe");
130 }
131
132 private void init() throws InterruptedException {
Eric Fellheimera39f8a92015-07-28 19:11:23 +0000133 long startTime = Profiler.nanoTimeMaybe();
Mark Schallerd7311e02015-07-07 16:36:09 +0000134 EvaluationResult<SkyValue> result =
135 graphFactory.prepareAndGet(universeScope, loadingPhaseThreads, eventHandler);
136 graph = result.getWalkableGraph();
Eric Fellheimera39f8a92015-07-28 19:11:23 +0000137 long duration = Profiler.nanoTimeMaybe() - startTime;
138 if (duration > 0) {
139 LOG.info("Spent " + (duration / 1000 / 1000) + " ms on evaluation and walkable graph");
140 }
Mark Schallerd7311e02015-07-07 16:36:09 +0000141
Mark Schaller28a676092015-08-27 18:11:04 +0000142 // The prepareAndGet call above evaluates a single PrepareDepsOfPatterns SkyKey.
143 // We expect to see either a single successfully evaluated value or a cycle in the result.
144 Collection<SkyValue> values = result.values();
145 if (!values.isEmpty()) {
146 Preconditions.checkState(values.size() == 1, "Universe query \"%s\" returned multiple"
147 + " values unexpectedly (%s values in result)", universeScope, values.size());
Mark Schallerd7311e02015-07-07 16:36:09 +0000148 PrepareDepsOfPatternsValue prepareDepsOfPatternsValue =
149 (PrepareDepsOfPatternsValue) Iterables.getOnlyElement(values);
150 universeTargetPatternKeys = prepareDepsOfPatternsValue.getTargetPatternKeys();
151 } else {
Mark Schaller28a676092015-08-27 18:11:04 +0000152 // No values in the result, so there must be an error. We expect the error to be a cycle.
153 boolean foundCycle = !Iterables.isEmpty(result.getError().getCycleInfo());
154 Preconditions.checkState(foundCycle, "Universe query \"%s\" failed with non-cycle error: %s",
155 universeScope, result.getError());
Mark Schallerd7311e02015-07-07 16:36:09 +0000156 universeTargetPatternKeys = ImmutableList.of();
157 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000158 }
159
160 @Override
161 public QueryEvalResult<Target> evaluateQuery(QueryExpression expr)
Janak Ramakrishnan89907392015-07-08 21:43:31 +0000162 throws QueryException, InterruptedException {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000163 // Some errors are reported as QueryExceptions and others as ERROR events (if --keep_going). The
164 // result is set to have an error iff there were errors emitted during the query, so we reset
165 // errors here.
166 eventHandler.resetErrors();
Janak Ramakrishnan89907392015-07-08 21:43:31 +0000167 init();
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000168 return super.evaluateQuery(expr);
169 }
170
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000171 private Map<Target, Collection<Target>> makeTargetsMap(Map<SkyKey, Iterable<SkyKey>> input) {
172 ImmutableMap.Builder<Target, Collection<Target>> result = ImmutableMap.builder();
173
174 for (Map.Entry<SkyKey, Target> entry : makeTargetsWithAssociations(input.keySet()).entrySet()) {
175 result.put(entry.getValue(), makeTargets(input.get(entry.getKey())));
176 }
177 return result.build();
178 }
179
180 private Map<Target, Collection<Target>> getRawFwdDeps(Iterable<Target> targets) {
181 return makeTargetsMap(graph.getDirectDeps(makeKeys(targets)));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000182 }
183
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000184 private Map<Target, Collection<Target>> getRawReverseDeps(Iterable<Target> targets) {
185 return makeTargetsMap(graph.getReverseDeps(makeKeys(targets)));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000186 }
187
188 private Set<Label> getAllowedDeps(Rule rule) {
Miguel Alcon Pinto4ffe28d2015-08-19 14:29:02 +0000189 Set<Label> allowedLabels = new HashSet<>(rule.getTransitions(dependencyFilter).values());
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000190 allowedLabels.addAll(rule.getVisibility().getDependencyLabels());
Marian Loburfdd788e2015-03-25 09:36:28 +0000191 // We should add deps from aspects, otherwise they are going to be filtered out.
192 allowedLabels.addAll(rule.getAspectLabelsSuperset(dependencyFilter));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000193 return allowedLabels;
194 }
195
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000196 private Collection<Target> filterFwdDeps(Target target, Collection<Target> rawFwdDeps) {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000197 if (!(target instanceof Rule)) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000198 return rawFwdDeps;
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000199 }
200 final Set<Label> allowedLabels = getAllowedDeps((Rule) target);
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000201 return Collections2.filter(rawFwdDeps,
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000202 new Predicate<Target>() {
203 @Override
204 public boolean apply(Target target) {
205 return allowedLabels.contains(target.getLabel());
206 }
207 });
208 }
209
Nathan Harmata3fae3662015-04-22 20:10:48 +0000210 @Override
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000211 public Collection<Target> getFwdDeps(Iterable<Target> targets) {
212 Set<Target> result = new HashSet<>();
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000213 for (Map.Entry<Target, Collection<Target>> entry : getRawFwdDeps(targets).entrySet()) {
214 result.addAll(filterFwdDeps(entry.getKey(), entry.getValue()));
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000215 }
216 return result;
217 }
218
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000219 @Override
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000220 public Collection<Target> getReverseDeps(Iterable<Target> targets) {
Miguel Alcon Pintob45e2622015-08-21 18:31:23 +0000221 Set<Target> result = CompactHashSet.create();
222 Map<Target, Collection<Target>> rawReverseDeps = getRawReverseDeps(targets);
223
224 CompactHashSet<Target> visited = CompactHashSet.create();
225
226 Set<Label> keys = CompactHashSet.create(Collections2.transform(rawReverseDeps.keySet(),
227 TARGET_LABEL_FUNCTION));
228 for (Collection<Target> parentCollection : rawReverseDeps.values()) {
229 for (Target parent : parentCollection) {
230 if (visited.add(parent)) {
231 if (parent instanceof Rule) {
232 for (Label label : getAllowedDeps((Rule) parent)) {
233 if (keys.contains(label)) {
234 result.add(parent);
235 }
236 }
237 } else {
238 result.add(parent);
239 }
240 }
241 }
Janak Ramakrishnan73dd2302015-06-16 17:04:25 +0000242 }
243 return result;
244 }
245
246 @Override
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000247 public Set<Target> getTransitiveClosure(Set<Target> targets) {
248 Set<Target> visited = new HashSet<>();
Janak Ramakrishnanb735d2f2015-06-16 17:46:36 +0000249 Collection<Target> current = targets;
250 while (!current.isEmpty()) {
251 Collection<Target> toVisit = Collections2.filter(current,
252 Predicates.not(Predicates.in(visited)));
253 current = getFwdDeps(toVisit);
254 visited.addAll(toVisit);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000255 }
Janak Ramakrishnanb735d2f2015-06-16 17:46:36 +0000256 return ImmutableSet.copyOf(visited);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000257 }
258
259 // Implemented with a breadth-first search.
260 @Override
261 public Set<Target> getNodesOnPath(Target from, Target to) {
262 // Tree of nodes visited so far.
263 Map<Target, Target> nodeToParent = new HashMap<>();
264 // Contains all nodes left to visit in a (LIFO) stack.
265 Deque<Target> toVisit = new ArrayDeque<>();
266 toVisit.add(from);
267 nodeToParent.put(from, null);
268 while (!toVisit.isEmpty()) {
269 Target current = toVisit.removeFirst();
270 if (to.equals(current)) {
271 return ImmutableSet.copyOf(Digraph.getPathToTreeNode(nodeToParent, to));
272 }
Janak Ramakrishnancda5b662015-06-18 23:46:36 +0000273 for (Target dep : getFwdDeps(ImmutableList.of(current))) {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000274 if (!nodeToParent.containsKey(dep)) {
275 nodeToParent.put(dep, current);
276 toVisit.addFirst(dep);
277 }
278 }
279 }
280 // Note that the only current caller of this method checks first to see if there is a path
281 // before calling this method. It is not clear what the return value should be here.
282 return null;
283 }
284
285 @Override
286 public Set<Target> getTargetsMatchingPattern(QueryExpression owner, String pattern)
287 throws QueryException {
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000288 Set<Target> targets = new LinkedHashSet<>(resolvedTargetPatterns.get(pattern));
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000289
290 // Sets.filter would be more convenient here, but can't deal with exceptions.
291 Iterator<Target> targetIterator = targets.iterator();
292 while (targetIterator.hasNext()) {
293 Target target = targetIterator.next();
294 if (!validateScope(target.getLabel(), strictScope)) {
295 targetIterator.remove();
296 }
297 }
298 return targets;
299 }
300
301 @Override
302 public Set<Target> getBuildFiles(QueryExpression caller, Set<Target> nodes)
303 throws QueryException {
304 Set<Target> dependentFiles = new LinkedHashSet<>();
305 Set<Package> seenPackages = new HashSet<>();
306 // Keep track of seen labels, to avoid adding a fake subinclude label that also exists as a
307 // real target.
308 Set<Label> seenLabels = new HashSet<>();
309
310 // Adds all the package definition files (BUILD files and build
311 // extensions) for package "pkg", to "buildfiles".
312 for (Target x : nodes) {
313 Package pkg = x.getPackage();
314 if (seenPackages.add(pkg)) {
315 addIfUniqueLabel(pkg.getBuildFile(), seenLabels, dependentFiles);
316 for (Label subinclude
317 : Iterables.concat(pkg.getSubincludeLabels(), pkg.getSkylarkFileDependencies())) {
318 addIfUniqueLabel(getSubincludeTarget(subinclude, pkg), seenLabels, dependentFiles);
319
320 // Also add the BUILD file of the subinclude.
321 try {
322 addIfUniqueLabel(getSubincludeTarget(
323 subinclude.getLocalTargetLabel("BUILD"), pkg), seenLabels, dependentFiles);
324 } catch (Label.SyntaxException e) {
325 throw new AssertionError("BUILD should always parse as a target name", e);
326 }
327 }
328 }
329 }
330 return dependentFiles;
331 }
332
333 private static void addIfUniqueLabel(Target node, Set<Label> labels, Set<Target> nodes) {
334 if (labels.add(node.getLabel())) {
335 nodes.add(node);
336 }
337 }
338
339 private static Target getSubincludeTarget(final Label label, Package pkg) {
Mark Schallerb817e3f2015-06-04 17:14:18 +0000340 return new FakeSubincludeTarget(label, pkg);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000341 }
342
343 @Override
344 public TargetAccessor<Target> getAccessor() {
345 return accessor;
346 }
347
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000348 private SkyKey getPackageKeyAndValidateLabel(Label label) throws QueryException {
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000349 // Can't use strictScope here because we are expecting a target back.
350 validateScope(label, true);
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000351 return PackageValue.key(label.getPackageIdentifier());
352 }
353
354 @Override
355 public Target getTarget(Label label) throws TargetNotFoundException, QueryException {
356 SkyKey packageKey = getPackageKeyAndValidateLabel(label);
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000357 if (!graph.exists(packageKey)) {
358 throw new QueryException(packageKey + " does not exist in graph");
359 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000360 try {
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000361 PackageValue packageValue = (PackageValue) graph.getValue(packageKey);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000362 if (packageValue != null) {
363 return packageValue.getPackage().getTarget(label.getName());
364 } else {
365 throw (NoSuchThingException) Preconditions.checkNotNull(
366 graph.getException(packageKey), label);
367 }
368 } catch (NoSuchThingException e) {
369 throw new TargetNotFoundException(e);
370 }
371 }
372
373 @Override
374 public void buildTransitiveClosure(QueryExpression caller, Set<Target> targets, int maxDepth)
375 throws QueryException {
376 // Everything has already been loaded, so here we just check for errors so that we can
377 // pre-emptively throw/report if needed.
Janak Ramakrishnanf6f0fcc2015-06-19 20:24:52 +0000378 for (Map.Entry<SkyKey, Exception> entry :
379 graph.getMissingAndExceptions(makeKeys(targets)).entrySet()) {
380 if (entry.getValue() == null) {
381 throw new QueryException(entry.getKey().argument() + " does not exist in graph");
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000382 }
Janak Ramakrishnanf6f0fcc2015-06-19 20:24:52 +0000383 reportBuildFileError(caller, entry.getValue().getMessage());
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000384 }
385 }
386
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000387 private Set<Target> filterTargetsNotInGraph(Set<Target> targets) {
388 Map<Target, SkyKey> map = Maps.toMap(targets, TARGET_TO_SKY_KEY);
389 Set<SkyKey> present = graph.getDoneValues(map.values()).keySet();
390 if (present.size() == targets.size()) {
391 // Optimize for case of all targets being in graph.
392 return targets;
393 }
394 return Maps.filterValues(map, Predicates.in(present)).keySet();
395 }
396
Nathan Harmata3fae3662015-04-22 20:10:48 +0000397 @Override
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000398 protected Map<String, Set<Target>> preloadOrThrow(
399 QueryExpression caller, Collection<String> patterns)
400 throws QueryException, TargetParsingException {
Nathan Harmata3fae3662015-04-22 20:10:48 +0000401 GraphBackedRecursivePackageProvider provider =
Mark Schallerd7311e02015-07-07 16:36:09 +0000402 new GraphBackedRecursivePackageProvider(graph, universeTargetPatternKeys);
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000403 Map<String, Set<Target>> result = Maps.newHashMapWithExpectedSize(patterns.size());
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000404 for (String pattern : patterns) {
405 SkyKey patternKey = TargetPatternValue.key(pattern,
406 TargetPatternEvaluator.DEFAULT_FILTERING_POLICY, parserPrefix);
Mark Schallerb889cf32015-03-17 20:55:30 +0000407
Mark Schaller248f0442015-05-19 17:59:36 +0000408 TargetPatternValue.TargetPatternKey targetPatternKey =
409 ((TargetPatternValue.TargetPatternKey) patternKey.argument());
Mark Schallerb889cf32015-03-17 20:55:30 +0000410
Mark Schaller895b3d22015-03-23 14:27:26 +0000411 TargetParsingException targetParsingException = null;
Mark Schallerb889cf32015-03-17 20:55:30 +0000412 if (graph.exists(patternKey)) {
Nathan Harmata3fae3662015-04-22 20:10:48 +0000413 // The graph already contains a value or exception for this target pattern, so we use it.
Mark Schallerb889cf32015-03-17 20:55:30 +0000414 TargetPatternValue value = (TargetPatternValue) graph.getValue(patternKey);
415 if (value != null) {
Janak Ramakrishnan36a2e832015-08-25 14:26:34 +0000416 result.put(
417 pattern, ImmutableSet.copyOf(makeTargetsFromLabels(value.getTargets().getTargets())));
Mark Schallerb889cf32015-03-17 20:55:30 +0000418 } else {
Janak Ramakrishnanf7ff6162015-06-02 20:20:31 +0000419 // Because the graph was always initialized via a keep_going build, we know that the
420 // exception stored here must be a TargetParsingException. Thus the comment in
421 // SkyframeTargetPatternEvaluator#parseTargetPatternKeys describing the situation in which
422 // the exception acceptance must be looser does not apply here.
Mark Schaller895b3d22015-03-23 14:27:26 +0000423 targetParsingException =
424 (TargetParsingException)
425 Preconditions.checkNotNull(graph.getException(patternKey), pattern);
Mark Schallerb889cf32015-03-17 20:55:30 +0000426 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000427 } else {
Mark Schallerb889cf32015-03-17 20:55:30 +0000428 // If the graph doesn't contain a value for this target pattern, try to directly evaluate
429 // it, by making use of packages already present in the graph.
Mark Schallerb889cf32015-03-17 20:55:30 +0000430 RecursivePackageProviderBackedTargetPatternResolver resolver =
431 new RecursivePackageProviderBackedTargetPatternResolver(provider, eventHandler,
Mark Schaller248f0442015-05-19 17:59:36 +0000432 targetPatternKey.getPolicy(), pkgPath);
Mark Schaller66b35f32015-05-19 21:19:37 +0000433 TargetPattern parsedPattern = targetPatternKey.getParsedPattern();
Mark Schallerb889cf32015-03-17 20:55:30 +0000434 try {
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000435 result.put(pattern, filterTargetsNotInGraph(parsedPattern.eval(resolver).getTargets()));
Mark Schaller895b3d22015-03-23 14:27:26 +0000436 } catch (TargetParsingException e) {
437 targetParsingException = e;
Mark Schallerb889cf32015-03-17 20:55:30 +0000438 } catch (InterruptedException e) {
439 throw new QueryException(e.getMessage());
440 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000441 }
Mark Schaller895b3d22015-03-23 14:27:26 +0000442
443 if (targetParsingException != null) {
444 if (!keepGoing) {
445 throw targetParsingException;
446 } else {
447 eventHandler.handle(Event.error("Evaluation of query \"" + caller + "\" failed: "
448 + targetParsingException.getMessage()));
Janak Ramakrishnancbe26342015-08-17 18:57:57 +0000449 result.put(pattern, ImmutableSet.<Target>of());
Mark Schaller895b3d22015-03-23 14:27:26 +0000450 }
451 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000452 }
453 return result;
454 }
455
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000456 private Collection<Target> makeTargets(Iterable<SkyKey> keys) {
457 return makeTargetsWithAssociations(keys).values();
458 }
459
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000460 private static final Function<SkyKey, Label> SKYKEY_TO_LABEL = new Function<SkyKey, Label>() {
461 @Nullable
462 @Override
463 public Label apply(SkyKey skyKey) {
464 SkyFunctionName functionName = skyKey.functionName();
Mark Schaller8ff5b3c2015-07-29 17:32:11 +0000465 if (!functionName.equals(SkyFunctions.TRANSITIVE_TRAVERSAL)) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000466 // Skip non-targets.
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000467 return null;
468 }
469 return (Label) skyKey.argument();
470 }
471 };
472
473 private Map<SkyKey, Target> makeTargetsWithAssociations(Iterable<SkyKey> keys) {
474 return makeTargetsWithAssociations(keys, SKYKEY_TO_LABEL);
475 }
476
477 private Collection<Target> makeTargetsFromLabels(Iterable<Label> labels) {
478 return makeTargetsWithAssociations(labels, Functions.<Label>identity()).values();
479 }
480
481 private <E> Map<E, Target> makeTargetsWithAssociations(Iterable<E> keys,
482 Function<E, Label> toLabel) {
483 Multimap<SkyKey, E> packageKeyToTargetKeyMap = ArrayListMultimap.create();
484 for (E key : keys) {
485 Label label = toLabel.apply(key);
486 if (label == null) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000487 continue;
488 }
489 try {
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000490 packageKeyToTargetKeyMap.put(getPackageKeyAndValidateLabel(label), key);
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000491 } catch (QueryException e) {
492 // Skip disallowed labels.
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000493 }
494 }
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000495 ImmutableMap.Builder<E, Target> result = ImmutableMap.builder();
Janak Ramakrishnanf6f0fcc2015-06-19 20:24:52 +0000496 Map<SkyKey, SkyValue> packageMap = graph.getDoneValues(packageKeyToTargetKeyMap.keySet());
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000497 for (Map.Entry<SkyKey, SkyValue> entry : packageMap.entrySet()) {
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000498 for (E targetKey : packageKeyToTargetKeyMap.get(entry.getKey())) {
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000499 try {
500 result.put(targetKey, ((PackageValue) entry.getValue()).getPackage()
Janak Ramakrishnanb5a541a2015-06-19 20:55:01 +0000501 .getTarget((toLabel.apply(targetKey)).getName()));
Janak Ramakrishnan36858732015-06-17 16:45:47 +0000502 } catch (NoSuchTargetException e) {
503 // Skip missing target.
504 }
505 }
506 }
507 return result.build();
508 }
509
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000510 private static final Function<Target, SkyKey> TARGET_TO_SKY_KEY =
511 new Function<Target, SkyKey>() {
512 @Override
513 public SkyKey apply(Target target) {
514 return TransitiveTraversalValue.key(target.getLabel());
515 }
516 };
517
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000518 private static Iterable<SkyKey> makeKeys(Iterable<Target> targets) {
Janak Ramakrishnana40e7b72015-08-20 20:06:16 +0000519 return Iterables.transform(targets, TARGET_TO_SKY_KEY);
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000520 }
521
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000522 @Override
523 public Target getOrCreate(Target target) {
524 return target;
525 }
Janak Ramakrishnan643063d2015-06-25 16:21:49 +0000526
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000527 /**
528 * Get SkyKeys for the FileValues for the given {@param pathFragments}. To do this, we look for a
529 * package lookup node for each path fragment, since package lookup nodes contain the "root" of a
530 * package. The returned SkyKeys correspond to FileValues that may not exist in the graph.
531 */
532 private Collection<SkyKey> getSkyKeysForFileFragments(Iterable<PathFragment> pathFragments) {
533 Set<SkyKey> result = new HashSet<>();
534 Multimap<PathFragment, PathFragment> currentToOriginal = ArrayListMultimap.create();
535 for (PathFragment pathFragment : pathFragments) {
536 currentToOriginal.put(pathFragment, pathFragment);
537 }
538 while (!currentToOriginal.isEmpty()) {
539 Map<SkyKey, PathFragment> keys = new HashMap<>();
540 for (PathFragment pathFragment : currentToOriginal.keySet()) {
541 keys.put(
542 PackageLookupValue.key(PackageIdentifier.createInDefaultRepo(pathFragment)),
543 pathFragment);
544 }
545 Map<SkyKey, SkyValue> lookupValues = graph.getDoneValues(keys.keySet());
546 for (Map.Entry<SkyKey, SkyValue> entry : lookupValues.entrySet()) {
547 PackageLookupValue packageLookupValue = (PackageLookupValue) entry.getValue();
548 PathFragment dir = keys.get(entry.getKey());
549 if (packageLookupValue.packageExists()) {
550 Collection<PathFragment> originalFiles = currentToOriginal.get(dir);
551 Preconditions.checkState(!originalFiles.isEmpty(), entry);
552 for (PathFragment fileName : originalFiles) {
553 result.add(
554 FileValue.key(RootedPath.toRootedPath(packageLookupValue.getRoot(), fileName)));
555 }
556 }
557 currentToOriginal.removeAll(dir);
558 }
559 Multimap<PathFragment, PathFragment> newCurrentToOriginal = ArrayListMultimap.create();
560 for (PathFragment pathFragment : currentToOriginal.keySet()) {
561 PathFragment parent = pathFragment.getParentDirectory();
562 if (parent != null) {
563 newCurrentToOriginal.putAll(parent, currentToOriginal.get(pathFragment));
564 }
565 }
566 currentToOriginal = newCurrentToOriginal;
567 }
568 return result;
569 }
570
571 /**
572 * Calculates the set of {@link Package} objects, represented as source file targets, that depend
573 * on the given list of BUILD files and subincludes (other files are filtered out).
574 */
575 @Nullable
576 Set<Target> getRBuildFiles(Collection<PathFragment> fileIdentifiers) {
577 Collection<SkyKey> files = getSkyKeysForFileFragments(fileIdentifiers);
578 Collection<SkyKey> current = graph.getDoneValues(files).keySet();
579 Set<SkyKey> resultKeys = CompactHashSet.create();
580 while (!current.isEmpty()) {
581 Collection<Iterable<SkyKey>> reverseDeps = graph.getReverseDeps(current).values();
582 current = new HashSet<>();
583 for (SkyKey rdep : Iterables.concat(reverseDeps)) {
584 if (rdep.functionName().equals(SkyFunctions.PACKAGE)) {
585 resultKeys.add(rdep);
586 } else if (!rdep.functionName().equals(SkyFunctions.PACKAGE_LOOKUP)) {
587 // Packages may depend on subpackages for existence, but we don't report them as rdeps.
588 current.add(rdep);
589 }
590 }
591 }
592 Map<SkyKey, SkyValue> packageValues = graph.getDoneValues(resultKeys);
593 if (packageValues.size() != resultKeys.size()) {
594 throw new IllegalStateException(
595 "Missing values: " + Sets.difference(resultKeys, packageValues.keySet()));
596 }
597 ImmutableSet.Builder<Target> result = ImmutableSet.builder();
598 for (SkyValue value : packageValues.values()) {
599 result.add(((PackageValue) value).getPackage().getBuildFile());
600 }
601 return result.build();
602 }
603
Janak Ramakrishnan643063d2015-06-25 16:21:49 +0000604 @Override
605 public Iterable<QueryFunction> getFunctions() {
606 return ImmutableList.<QueryFunction>builder()
Janak Ramakrishnand802d5b2015-08-20 21:05:46 +0000607 .addAll(super.getFunctions())
608 .add(new AllRdepsFunction())
609 .add(new RBuildFilesFunction())
610 .build();
Janak Ramakrishnan643063d2015-06-25 16:21:49 +0000611 }
Janak Ramakrishnane72d5222015-02-26 17:09:18 +0000612}