Add a mechanism for bounding the number of Packages SkyQueryEnvironment's expensive parallel operations can operate on at once.
--
MOS_MIGRATED_REVID=138779172
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePackageProviderBackedTargetPatternResolver.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePackageProviderBackedTargetPatternResolver.java
index 62926c8..0071098 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePackageProviderBackedTargetPatternResolver.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePackageProviderBackedTargetPatternResolver.java
@@ -29,6 +29,7 @@
import com.google.devtools.build.lib.cmdline.TargetParsingException;
import com.google.devtools.build.lib.cmdline.TargetPatternResolver;
import com.google.devtools.build.lib.concurrent.MoreFutures;
+import com.google.devtools.build.lib.concurrent.MultisetSemaphore;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
@@ -65,16 +66,19 @@
private final EventHandler eventHandler;
private final FilteringPolicy policy;
private final ExecutorService executor;
+ private final MultisetSemaphore<PackageIdentifier> packageSemaphore;
public RecursivePackageProviderBackedTargetPatternResolver(
RecursivePackageProvider recursivePackageProvider,
EventHandler eventHandler,
FilteringPolicy policy,
- ExecutorService executor) {
+ ExecutorService executor,
+ MultisetSemaphore<PackageIdentifier> packageSemaphore) {
this.recursivePackageProvider = recursivePackageProvider;
this.eventHandler = eventHandler;
this.policy = policy;
this.executor = executor;
+ this.packageSemaphore = packageSemaphore;
}
@Override
@@ -215,22 +219,28 @@
tasks.add(executor.submit(new Callable<Void>() {
@Override
public Void call() throws E, TargetParsingException, InterruptedException {
- Iterable<ResolvedTargets<Target>> resolvedTargets =
- bulkGetTargetsInPackage(originalPattern, pkgIdBatch, NO_FILTER).values();
- List<Target> filteredTargets = new ArrayList<>(calculateSize(resolvedTargets));
- for (ResolvedTargets<Target> targets : resolvedTargets) {
- for (Target target : targets.getTargets()) {
- // Perform the no-targets-found check before applying the filtering policy
- // so we only return the error if the input directory's subtree really
- // contains no targets.
- foundTarget.set(true);
- if (actualPolicy.shouldRetain(target, false)) {
- filteredTargets.add(target);
+ ImmutableSet<PackageIdentifier> pkgIdBatchSet = ImmutableSet.copyOf(pkgIdBatch);
+ packageSemaphore.acquireAll(pkgIdBatchSet);
+ try {
+ Iterable<ResolvedTargets<Target>> resolvedTargets =
+ bulkGetTargetsInPackage(originalPattern, pkgIdBatch, NO_FILTER).values();
+ List<Target> filteredTargets = new ArrayList<>(calculateSize(resolvedTargets));
+ for (ResolvedTargets<Target> targets : resolvedTargets) {
+ for (Target target : targets.getTargets()) {
+ // Perform the no-targets-found check before applying the filtering policy
+ // so we only return the error if the input directory's subtree really
+ // contains no targets.
+ foundTarget.set(true);
+ if (actualPolicy.shouldRetain(target, false)) {
+ filteredTargets.add(target);
+ }
}
}
- }
- synchronized (callbackLock) {
- callback.process(filteredTargets);
+ synchronized (callbackLock) {
+ callback.process(filteredTargets);
+ }
+ } finally {
+ packageSemaphore.releaseAll(pkgIdBatchSet);
}
return null;
}
