blob: f65ed6ae896348cf9aef44017b7360ef4526fb19 [file] [log] [blame]
// Copyright 2015 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import static com.google.common.util.concurrent.Futures.immediateCancelledFuture;
import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.Uninterruptibles;
import com.google.devtools.build.lib.cmdline.BatchCallback;
import com.google.devtools.build.lib.cmdline.BatchCallback.SafeBatchCallback;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.cmdline.QueryExceptionMarkerInterface;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.cmdline.ResolvedTargets;
import com.google.devtools.build.lib.cmdline.TargetParsingException;
import com.google.devtools.build.lib.cmdline.TargetPatternResolver;
import com.google.devtools.build.lib.concurrent.MultisetSemaphore;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.io.InconsistentFilesystemException;
import com.google.devtools.build.lib.io.ProcessPackageDirectoryException;
import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.NoSuchThingException;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.pkgcache.FilteringPolicies;
import com.google.devtools.build.lib.pkgcache.FilteringPolicy;
import com.google.devtools.build.lib.pkgcache.RecursivePackageProvider;
import com.google.devtools.build.lib.pkgcache.TargetPatternResolverUtil;
import com.google.devtools.build.lib.query2.engine.QueryException;
import com.google.devtools.build.lib.server.FailureDetails.TargetPatterns;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import javax.annotation.Nullable;
/** A {@link TargetPatternResolver} backed by a {@link RecursivePackageProvider}. */
@ThreadCompatible
public final class RecursivePackageProviderBackedTargetPatternResolver
extends TargetPatternResolver<Target> {
// TODO(janakr): Move this to a more generic place and unify with SkyQueryEnvironment's value?
static final int MAX_PACKAGES_BULK_GET = 1000;
protected final FilteringPolicy policy;
private final RecursivePackageProvider recursivePackageProvider;
private final ExtendedEventHandler eventHandler;
private final MultisetSemaphore<PackageIdentifier> packageSemaphore;
private final PackageIdentifierBatchingCallback.Factory packageIdentifierBatchingCallbackFactory;
public RecursivePackageProviderBackedTargetPatternResolver(
RecursivePackageProvider recursivePackageProvider,
ExtendedEventHandler eventHandler,
FilteringPolicy policy,
MultisetSemaphore<PackageIdentifier> packageSemaphore,
PackageIdentifierBatchingCallback.Factory packageIdentifierBatchingCallbackFactory) {
this.recursivePackageProvider = recursivePackageProvider;
this.eventHandler = eventHandler;
this.policy = policy;
this.packageSemaphore = packageSemaphore;
this.packageIdentifierBatchingCallbackFactory = packageIdentifierBatchingCallbackFactory;
}
@Override
public void warn(String msg) {
eventHandler.handle(Event.warn(msg));
}
/**
* Gets a {@link Package} from the {@link RecursivePackageProvider}. May return a {@link Package}
* that has errors.
*/
private Package getPackage(PackageIdentifier pkgIdentifier)
throws NoSuchPackageException, InterruptedException {
return recursivePackageProvider.getPackage(eventHandler, pkgIdentifier);
}
private Map<PackageIdentifier, Package> bulkGetPackages(Iterable<PackageIdentifier> pkgIds)
throws NoSuchPackageException, InterruptedException {
return recursivePackageProvider.bulkGetPackages(pkgIds);
}
@Override
@Nullable
public Target getTargetOrNull(Label label)
throws InterruptedException, InconsistentFilesystemException {
try {
if (!isPackage(label.getPackageIdentifier())) {
return null;
}
return recursivePackageProvider.getTarget(eventHandler, label);
} catch (NoSuchThingException e) {
return null;
}
}
@Override
public ResolvedTargets<Target> getExplicitTarget(Label label)
throws TargetParsingException, InterruptedException {
try {
Target target = recursivePackageProvider.getTarget(eventHandler, label);
return policy.shouldRetain(target, true)
? ResolvedTargets.of(target)
: ResolvedTargets.empty();
} catch (NoSuchThingException e) {
throw new TargetParsingException(e.getMessage(), e, e.getDetailedExitCode());
}
}
@Override
public Collection<Target> getTargetsInPackage(
String originalPattern, PackageIdentifier packageIdentifier, boolean rulesOnly)
throws TargetParsingException, InterruptedException {
FilteringPolicy actualPolicy =
rulesOnly ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy) : policy;
try {
Package pkg = getPackage(packageIdentifier);
if (pkg.containsErrors()) {
eventHandler.handle(
Event.error(
"package contains errors: "
+ pkg.getNameFragment()
+ ": "
+ pkg.getFailureDetail().getMessage()));
}
return TargetPatternResolverUtil.resolvePackageTargets(pkg, actualPolicy);
} catch (NoSuchThingException e) {
String message =
TargetPatternResolverUtil.getParsingErrorMessage(e.getMessage(), originalPattern);
throw new TargetParsingException(message, e, e.getDetailedExitCode());
}
}
private Map<PackageIdentifier, Collection<Target>> bulkGetTargetsInPackage(
String originalPattern, Iterable<PackageIdentifier> pkgIds, FilteringPolicy policy)
throws InterruptedException {
try {
Map<PackageIdentifier, Package> pkgs = bulkGetPackages(pkgIds);
if (pkgs.size() != Iterables.size(pkgIds)) {
throw new IllegalStateException(
"Bulk package retrieval missing results: "
+ Sets.difference(ImmutableSet.copyOf(pkgIds), pkgs.keySet()));
}
ImmutableMap.Builder<PackageIdentifier, Collection<Target>> result = ImmutableMap.builder();
for (PackageIdentifier pkgId : pkgIds) {
Package pkg = pkgs.get(pkgId);
if (pkg.containsErrors()) {
eventHandler.handle(
Event.error(
"package contains errors: "
+ pkg.getNameFragment()
+ ": "
+ pkg.getFailureDetail().getMessage()));
}
result.put(pkgId, TargetPatternResolverUtil.resolvePackageTargets(pkg, policy));
}
return result.buildOrThrow();
} catch (NoSuchThingException e) {
String message =
TargetPatternResolverUtil.getParsingErrorMessage(e.getMessage(), originalPattern);
throw new IllegalStateException(
"Mismatch: Expected given pkgIds to correspond to valid Packages. " + message, e);
}
}
@Override
public boolean isPackage(PackageIdentifier packageIdentifier)
throws InterruptedException, InconsistentFilesystemException {
return recursivePackageProvider.isPackage(eventHandler, packageIdentifier);
}
@Override
public String getTargetKind(Target target) {
return target.getTargetKind();
}
@Override
public <E extends Exception & QueryExceptionMarkerInterface> void findTargetsBeneathDirectory(
final RepositoryName repository,
final String originalPattern,
String directory,
boolean rulesOnly,
ImmutableSet<PathFragment> forbiddenSubdirectories,
ImmutableSet<PathFragment> excludedSubdirectories,
BatchCallback<Target, E> callback,
Class<E> exceptionClass)
throws TargetParsingException, E, InterruptedException, ProcessPackageDirectoryException {
ListenableFuture<Void> future;
try {
future =
findTargetsBeneathDirectoryAsyncImpl(
repository,
originalPattern,
directory,
rulesOnly,
forbiddenSubdirectories,
excludedSubdirectories,
callback,
MoreExecutors.newDirectExecutorService());
} catch (QueryException e) {
Throwables.propagateIfPossible(e, exceptionClass);
throw new IllegalStateException(e);
} catch (NoSuchPackageException e) {
// Can happen during a Skyframe no-keep-going evaluation.
throw new TargetParsingException(
"error loading package under directory '" + directory + "': " + e.getMessage(),
e,
e.getDetailedExitCode());
}
if (!isSuccessful(future)) {
// Don't get the future if it finished successfully: all that will do is throw an
// interrupted exception if this thread was interrupted, but that's not helpful for a done
// future.
try {
future.get();
} catch (ExecutionException e) {
Throwables.propagateIfPossible(e.getCause(), InterruptedException.class, exceptionClass);
throw new IllegalStateException(e.getCause());
}
}
}
@Override
public <E extends Exception & QueryExceptionMarkerInterface>
ListenableFuture<Void> findTargetsBeneathDirectoryAsync(
RepositoryName repository,
String originalPattern,
String directory,
boolean rulesOnly,
ImmutableSet<PathFragment> forbiddenSubdirectories,
ImmutableSet<PathFragment> excludedSubdirectories,
BatchCallback<Target, E> callback,
Class<E> exceptionClass,
ListeningExecutorService executor) {
try {
return findTargetsBeneathDirectoryAsyncImpl(
repository,
originalPattern,
directory,
rulesOnly,
forbiddenSubdirectories,
excludedSubdirectories,
callback,
executor);
} catch (TargetParsingException e) {
return immediateFailedFuture(e);
} catch (InterruptedException e) {
return immediateCancelledFuture();
} catch (ProcessPackageDirectoryException | NoSuchPackageException e) {
throw new IllegalStateException(
"Async find targets beneath directory isn't called from within Skyframe: traversing "
+ directory
+ " for "
+ originalPattern,
e);
} catch (QueryException e) {
if (exceptionClass.isInstance(e)) {
return immediateFailedFuture(e);
}
throw new IllegalStateException(e);
}
}
/**
* The returned future may throw {@link QueryException} (if {@code E} is {@link QueryException})
* or {@link InterruptedException} on retrieval, but no other exceptions.
*/
private <E extends Exception & QueryExceptionMarkerInterface>
ListenableFuture<Void> findTargetsBeneathDirectoryAsyncImpl(
RepositoryName repository,
String pattern,
String directory,
boolean rulesOnly,
ImmutableSet<PathFragment> forbiddenSubdirectories,
ImmutableSet<PathFragment> excludedSubdirectories,
BatchCallback<Target, E> callback,
ListeningExecutorService executor)
throws TargetParsingException, QueryException, InterruptedException,
ProcessPackageDirectoryException, NoSuchPackageException {
FilteringPolicy actualPolicy =
rulesOnly ? FilteringPolicies.and(FilteringPolicies.RULES_ONLY, policy) : policy;
ArrayList<ListenableFuture<Void>> futures = new ArrayList<>();
SafeBatchCallback<PackageIdentifier> getPackageTargetsCallback =
(pkgIdBatch) ->
futures.add(
executor.submit(
new GetTargetsInPackagesTask<>(pkgIdBatch, pattern, actualPolicy, callback)));
PathFragment pathFragment = TargetPatternResolverUtil.getPathFragment(directory);
try (PackageIdentifierBatchingCallback batchingCallback =
packageIdentifierBatchingCallbackFactory.create(
getPackageTargetsCallback, MAX_PACKAGES_BULK_GET)) {
recursivePackageProvider.streamPackagesUnderDirectory(
batchingCallback,
eventHandler,
repository,
pathFragment,
forbiddenSubdirectories,
excludedSubdirectories);
}
if (futures.isEmpty()) {
throw new TargetParsingException(
"no targets found beneath '" + pathFragment + "'", TargetPatterns.Code.TARGETS_MISSING);
}
return Futures.whenAllSucceed(futures).call(() -> null, directExecutor());
}
/**
* Task to get all matching targets in the given packages, filter them, and pass them to the
* target batch callback.
*/
private class GetTargetsInPackagesTask<E extends Exception & QueryExceptionMarkerInterface>
implements Callable<Void> {
private final Iterable<PackageIdentifier> packageIdentifiers;
private final String originalPattern;
private final FilteringPolicy actualPolicy;
private final BatchCallback<Target, E> callback;
GetTargetsInPackagesTask(
Iterable<PackageIdentifier> packageIdentifiers,
String originalPattern,
FilteringPolicy actualPolicy,
BatchCallback<Target, E> callback) {
this.packageIdentifiers = packageIdentifiers;
this.originalPattern = originalPattern;
this.actualPolicy = actualPolicy;
this.callback = callback;
}
@Override
public Void call() throws E, InterruptedException {
ImmutableSet<PackageIdentifier> pkgIdBatchSet = ImmutableSet.copyOf(packageIdentifiers);
packageSemaphore.acquireAll(pkgIdBatchSet);
try {
Iterable<Collection<Target>> resolvedTargets =
RecursivePackageProviderBackedTargetPatternResolver.this
.bulkGetTargetsInPackage(originalPattern, packageIdentifiers, actualPolicy)
.values();
List<Target> filteredTargets = new ArrayList<>(calculateSize(resolvedTargets));
for (Collection<Target> targets : resolvedTargets) {
filteredTargets.addAll(targets);
}
// TODO(b/121277360): Invoking the callback while holding onto the package
// semaphore can lead to deadlocks.
//
// Also, if the semaphore has a small count, acquireAll can also lead to problems if we
// don't batch appropriately. Note: We default to an unbounded semaphore for SkyQuery.
//
// TODO(b/168142585): Make this code strictly correct in the situation where the semaphore
// is bounded.
callback.process(filteredTargets);
} finally {
packageSemaphore.releaseAll(pkgIdBatchSet);
}
return null;
}
}
private static <T> int calculateSize(Iterable<Collection<T>> resolvedTargets) {
int size = 0;
for (Collection<T> targets : resolvedTargets) {
size += targets.size();
}
return size;
}
/** Inspired by not-yet-open-source futures code. */
private static boolean isSuccessful(Future<?> future) {
if (future.isDone() && !future.isCancelled()) {
try {
Uninterruptibles.getUninterruptibly(future);
return true;
} catch (ExecutionException | RuntimeException e) {
// Fall through.
}
}
return false;
}
}