Add the option --experimental_graphless_query
This option selects a new query implementation that does not make a copy
of the graph, and therefore does not support ordering in the same way
that normal query does. It can only be used with --order_output=no.
PiperOrigin-RevId: 251625161
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java
index a5b4150..40121ac 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java
@@ -102,7 +102,9 @@
Lists.<String>newArrayList(),
threadsOption.threads,
EnumSet.noneOf(Setting.class),
- /* useForkJoinPool= */ false);
+ // TODO(ulfjack): flip both these flags for improved performance.
+ /* useForkJoinPool= */ false,
+ /* useGraphlessQuery= */ false);
// 1. Parse query:
QueryExpression expr;
diff --git a/src/main/java/com/google/devtools/build/lib/query2/QueryEnvironmentFactory.java b/src/main/java/com/google/devtools/build/lib/query2/QueryEnvironmentFactory.java
index f78fcc6..bbf95159 100644
--- a/src/main/java/com/google/devtools/build/lib/query2/QueryEnvironmentFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/query2/QueryEnvironmentFactory.java
@@ -27,6 +27,7 @@
import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting;
import com.google.devtools.build.lib.query2.query.BlazeQueryEnvironment;
+import com.google.devtools.build.lib.query2.query.GraphlessBlazeQueryEnvironment;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.skyframe.WalkableGraph.WalkableGraphFactory;
import java.util.List;
@@ -54,7 +55,8 @@
Iterable<QueryFunction> extraFunctions,
@Nullable PathPackageLocator packagePath,
boolean blockUniverseEvaluationErrors,
- boolean useForkJoinPool) {
+ boolean useForkJoinPool,
+ boolean useGraphlessQuery) {
Preconditions.checkNotNull(universeScope);
if (canUseSkyQuery(orderedResults, universeScope, packagePath, strictScope, labelFilter)) {
return new SkyQueryEnvironment(
@@ -68,6 +70,21 @@
universeScope,
packagePath,
blockUniverseEvaluationErrors);
+ } else if (useGraphlessQuery) {
+ return new GraphlessBlazeQueryEnvironment(
+ transitivePackageLoader,
+ targetProvider,
+ cachingPackageLocator,
+ targetPatternPreloader,
+ relativeWorkingDirectory,
+ keepGoing,
+ strictScope,
+ loadingPhaseThreads,
+ labelFilter,
+ eventHandler,
+ settings,
+ extraFunctions,
+ useForkJoinPool);
} else {
return new BlazeQueryEnvironment(
transitivePackageLoader,
diff --git a/src/main/java/com/google/devtools/build/lib/query2/query/output/OutputFormatter.java b/src/main/java/com/google/devtools/build/lib/query2/query/output/OutputFormatter.java
index 04deae5..80c0ae7 100644
--- a/src/main/java/com/google/devtools/build/lib/query2/query/output/OutputFormatter.java
+++ b/src/main/java/com/google/devtools/build/lib/query2/query/output/OutputFormatter.java
@@ -18,6 +18,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
+import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
@@ -118,6 +119,13 @@
return Streams.stream(formatters).map(OutputFormatter::getName).collect(joining(", "));
}
+ public static String streamingFormatterNames(Iterable<OutputFormatter> formatters) {
+ return Streams.stream(formatters)
+ .filter(Predicates.instanceOf(StreamedFormatter.class))
+ .map(OutputFormatter::getName)
+ .collect(joining(", "));
+ }
+
/**
* Returns the output formatter for the specified command-line options.
*/
diff --git a/src/main/java/com/google/devtools/build/lib/query2/query/output/QueryOptions.java b/src/main/java/com/google/devtools/build/lib/query2/query/output/QueryOptions.java
index f0da529..8165148 100644
--- a/src/main/java/com/google/devtools/build/lib/query2/query/output/QueryOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/query2/query/output/QueryOptions.java
@@ -197,6 +197,17 @@
help = "If true, uses a ForkJoinPool instead of a PriorityBlockingQueue for Skyframe.")
public boolean useForkJoinPool;
+ @Option(
+ name = "experimental_graphless_query",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.QUERY,
+ effectTags = {OptionEffectTag.BUILD_FILE_SEMANTICS, OptionEffectTag.EAGERNESS_TO_EXIT},
+ help =
+ "If true, uses a Query implementation that does not make a copy of the graph. The new"
+ + " implementation only supports --order_output=no, as well as only a subset of"
+ + " output formatters.")
+ public boolean useGraphlessQuery;
+
/** Ugly workaround since line terminator option default has to be constant expression. */
public String getLineTerminator() {
if (lineTerminatorNull) {
diff --git a/src/main/java/com/google/devtools/build/lib/rules/genquery/GenQuery.java b/src/main/java/com/google/devtools/build/lib/rules/genquery/GenQuery.java
index 77fc9e8..d31fc28 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/genquery/GenQuery.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/genquery/GenQuery.java
@@ -344,7 +344,8 @@
/*extraFunctions=*/ ImmutableList.of(),
/*packagePath=*/ null,
/*blockUniverseEvaluationErrors=*/ false,
- /*useForkJoinPool=*/ false);
+ /*useForkJoinPool=*/ false,
+ /*useGraphlessQuery=*/ false);
QueryExpression expr = QueryExpression.parse(query, queryEnvironment);
formatter.verifyCompatible(queryEnvironment, expr);
targets = QueryUtil.newOrderedAggregateAllOutputFormatterCallback(queryEnvironment);
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java
index 10a775d..83a713d 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java
@@ -203,6 +203,7 @@
ImmutableList.<String>of(),
loadingPhaseThreads,
settings,
- /* useForkJoinPool= */ false);
+ /* useForkJoinPool= */ false,
+ /*useGraphlessQuery=*/ false);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryEnvironmentBasedCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryEnvironmentBasedCommand.java
index 2d0f8a8..9d46140 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryEnvironmentBasedCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryEnvironmentBasedCommand.java
@@ -139,6 +139,16 @@
Set<Setting> settings = queryOptions.toSettings();
boolean streamResults = QueryOutputUtils.shouldStreamResults(queryOptions, formatter);
+ if (queryOptions.useGraphlessQuery && !streamResults) {
+ env.getReporter()
+ .handle(
+ Event.error(
+ String.format(
+ "--experimental_graphless_query requires --order_output=no and an --output"
+ + " option that supports streaming; valid values are: %s",
+ OutputFormatter.streamingFormatterNames(formatters))));
+ return BlazeCommandResult.exitCode(ExitCode.COMMAND_LINE_ERROR);
+ }
try (QueryRuntimeHelper queryRuntimeHelper =
env.getRuntime().getQueryRuntimeHelperFactory().create(env)) {
@@ -151,7 +161,8 @@
queryOptions.universeScope,
options.getOptions(LoadingPhaseThreadsOption.class).threads,
settings,
- queryOptions.useForkJoinPool)) {
+ queryOptions.useForkJoinPool,
+ queryOptions.useGraphlessQuery)) {
result =
doQuery(
query, env, queryOptions, streamResults, formatter, queryEnv, queryRuntimeHelper);
@@ -200,7 +211,8 @@
List<String> universeScope,
int loadingPhaseThreads,
Set<Setting> settings,
- boolean useForkJoinPool) {
+ boolean useForkJoinPool,
+ boolean useGraphlessQuery) {
WalkableGraph walkableGraph =
SkyframeExecutorWrappingWalkableGraph.of(env.getSkyframeExecutor());
@@ -235,6 +247,7 @@
env.getRuntime().getQueryFunctions(),
env.getPackageManager().getPackagePath(),
/*blockUniverseEvaluationErrors=*/ false,
- useForkJoinPool);
+ useForkJoinPool,
+ useGraphlessQuery);
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/query2/engine/GraphlessQueryTest.java b/src/test/java/com/google/devtools/build/lib/query2/engine/GraphlessQueryTest.java
index d4ee02e..b60ebfd 100644
--- a/src/test/java/com/google/devtools/build/lib/query2/engine/GraphlessQueryTest.java
+++ b/src/test/java/com/google/devtools/build/lib/query2/engine/GraphlessQueryTest.java
@@ -113,7 +113,8 @@
Iterable<QueryFunction> extraFunctions,
@Nullable PathPackageLocator packagePath,
boolean blockUniverseEvaluationErrors,
- boolean useForkJoinPool) {
+ boolean useForkJoinPool,
+ boolean useGraphlessQuery) {
return new GraphlessBlazeQueryEnvironment(
transitivePackageLoader,
targetProvider,
diff --git a/src/test/java/com/google/devtools/build/lib/query2/engine/SkyframeQueryHelper.java b/src/test/java/com/google/devtools/build/lib/query2/engine/SkyframeQueryHelper.java
index 16db39b..b825fbf 100644
--- a/src/test/java/com/google/devtools/build/lib/query2/engine/SkyframeQueryHelper.java
+++ b/src/test/java/com/google/devtools/build/lib/query2/engine/SkyframeQueryHelper.java
@@ -200,7 +200,8 @@
getExtraQueryFunctions(),
pkgManager.getPackagePath(),
blockUniverseEvaluationErrors,
- /*useForkJoinPool=*/ false);
+ /*useForkJoinPool=*/ false,
+ /*useGraphlessQuery=*/ false);
}
protected abstract Iterable<QueryFunction> getExtraQueryFunctions();
