RELNOTES: --keep_incrementality_data flag allows Bazel servers to be run in memory-saving non-incremental mode independent of --batch and --discard_analysis_cache.
A command run with --nokeep_incrementality_data will discard data that would be needed for incremental builds. Subsequent commands can be sent to the same server, but they will not get the benefit of incrementality from this command. However, if --keep_incrementality_data is specified on a subsequent command, the commands after that will get the benefits of incrementality.
There are two benefits to not being dependent on --batch. First, this allows Bazel servers to be run in extreme memory-saving mode without the startup penalties (JVM startup, JITting) that --batch execution imposes. Second, this allows Bazel developers to inspect the state of a Bazel server after an extreme memory-saving build.
In order to avoid discarding data unnecessarily (for instance, on a "bazel info used-heap-size-after-gc" or "bazel dump --skyframe=summary") the actual resetting of the graph is done lazily, right before its use in SequencedSkyframeExecutor#sync. This is morally a partial rollback of https://github.com/bazelbuild/bazel/commit/98cd82cbdcac7c48164a611c5a9aa8fc2f1720ef.
For now, our tests specify all of the flags. After this change sticks, I plan to get rid of the --batch flag from these tests, which should allow for some clean-ups. Eventually --batch and --discard_analysis_cache may not imply that we don't keep incremental state: we can require that it be specified explicitly.
PiperOrigin-RevId: 175335075
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index c6e5649..b107cba 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -436,6 +436,7 @@
"//src/main/java/com/google/devtools/build/lib:transitive-info-provider",
],
deps = [
+ ":build-request-options",
"//src/main/java/com/google/devtools/build/lib:base-util",
"//src/main/java/com/google/devtools/build/lib:events",
"//src/main/java/com/google/devtools/build/lib:exitcode-external",
@@ -961,6 +962,18 @@
)
java_library(
+ name = "build-request-options",
+ srcs = ["buildtool/BuildRequestOptions.java"],
+ deps = [
+ "//src/main/java/com/google/devtools/build/lib:util",
+ "//src/main/java/com/google/devtools/build/lib/actions",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
+ "//src/main/java/com/google/devtools/common/options",
+ "//third_party:guava",
+ ],
+)
+
+java_library(
name = "runtime",
srcs = glob(
[
@@ -971,8 +984,10 @@
"buildtool/*.java",
"buildtool/buildevent/*.java",
],
+ exclude = ["buildtool/BuildRequestOptions.java"],
),
deps = [
+ ":build-request-options",
"//src/main/java/com/google/devtools/build/docgen:docgen_javalib",
"//src/main/java/com/google/devtools/build/lib:build-base",
"//src/main/java/com/google/devtools/build/lib:events",
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java
index 1780b1e..fb9a66a 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java
@@ -15,6 +15,7 @@
package com.google.devtools.build.lib.analysis.config;
import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.buildtool.BuildRequestOptions;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import java.util.HashMap;
import java.util.List;
@@ -24,7 +25,7 @@
*
* <p>The target configuration is used for all targets specified on the command line. Multiple
* target configurations are possible because of settings like {@link
- * com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions#multiCpus}.
+ * BuildRequestOptions#multiCpus}.
*
* <p>The host configuration is used for tools that are executed during the build, e. g, compilers.
*/
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java
index c6b4c75..6a489cc 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java
@@ -13,7 +13,6 @@
// limitations under the License.
package com.google.devtools.build.lib.buildtool;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheBuilder;
@@ -21,7 +20,6 @@
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedSet;
-import com.google.devtools.build.lib.actions.LocalHostCapacity;
import com.google.devtools.build.lib.analysis.BuildView;
import com.google.devtools.build.lib.analysis.OutputGroupProvider;
import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
@@ -33,23 +31,13 @@
import com.google.devtools.build.lib.runtime.BlazeCommandEventHandler;
import com.google.devtools.build.lib.util.OptionsUtils;
import com.google.devtools.build.lib.util.io.OutErr;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.common.options.Converters;
-import com.google.devtools.common.options.Converters.RangeConverter;
-import com.google.devtools.common.options.Option;
-import com.google.devtools.common.options.OptionDocumentationCategory;
-import com.google.devtools.common.options.OptionEffectTag;
-import com.google.devtools.common.options.OptionMetadataTag;
import com.google.devtools.common.options.OptionsBase;
import com.google.devtools.common.options.OptionsClassProvider;
-import com.google.devtools.common.options.OptionsParsingException;
import com.google.devtools.common.options.OptionsProvider;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
-import java.util.logging.Logger;
-import java.util.regex.Pattern;
/**
* A BuildRequest represents a single invocation of the build tool by a user.
@@ -58,440 +46,6 @@
* as --keep_going, --jobs, etc.
*/
public class BuildRequest implements OptionsClassProvider {
- private static final Logger logger = Logger.getLogger(BuildRequest.class.getName());
-
- /**
- * Options interface--can be used to parse command-line arguments.
- *
- * <p>See also ExecutionOptions; from the user's point of view, there's no
- * qualitative difference between these two sets of options.
- */
- public static class BuildRequestOptions extends OptionsBase {
-
- /* "Execution": options related to the execution of a build: */
-
- @Option(
- name = "jobs",
- abbrev = 'j',
- defaultValue = "auto",
- category = "strategy",
- documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
- effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS, OptionEffectTag.EXECUTION},
- converter = JobsConverter.class,
- help =
- "The number of concurrent jobs to run. 0 means build sequentially."
- + " \"auto\" means to use a reasonable value derived from the machine's hardware"
- + " profile (e.g. the number of processors). Values above "
- + MAX_JOBS
- + " are not allowed, and values above "
- + JOBS_TOO_HIGH_WARNING
- + " may cause memory issues."
- )
- public int jobs;
-
- @Option(
- name = "progress_report_interval",
- defaultValue = "0",
- category = "verbosity",
- documentationCategory = OptionDocumentationCategory.LOGGING,
- effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
- converter = ProgressReportIntervalConverter.class,
- help =
- "The number of seconds to wait between two reports on still running jobs. The "
- + "default value 0 means to use the default 10:30:60 incremental algorithm."
- )
- public int progressReportInterval;
-
- @Option(
- name = "explain",
- defaultValue = "null",
- category = "verbosity",
- documentationCategory = OptionDocumentationCategory.LOGGING,
- effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
- converter = OptionsUtils.PathFragmentConverter.class,
- help =
- "Causes the build system to explain each executed step of the "
- + "build. The explanation is written to the specified log file."
- )
- public PathFragment explanationPath;
-
- @Option(
- name = "verbose_explanations",
- defaultValue = "false",
- category = "verbosity",
- documentationCategory = OptionDocumentationCategory.LOGGING,
- effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
- help =
- "Increases the verbosity of the explanations issued if --explain is enabled. "
- + "Has no effect if --explain is not enabled."
- )
- public boolean verboseExplanations;
-
- @Option(
- name = "output_filter",
- converter = Converters.RegexPatternConverter.class,
- defaultValue = "null",
- category = "flags",
- documentationCategory = OptionDocumentationCategory.LOGGING,
- effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
- help = "Only shows warnings for rules with a name matching the provided regular expression."
- )
- public Pattern outputFilter;
-
- @Deprecated
- @Option(
- name = "dump_makefile",
- defaultValue = "false",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {OptionEffectTag.NO_OP},
- metadataTags = {OptionMetadataTag.DEPRECATED},
- help = "this flag has no effect."
- )
- public boolean dumpMakefile;
-
- @Deprecated
- @Option(
- name = "dump_action_graph",
- defaultValue = "false",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {OptionEffectTag.NO_OP},
- metadataTags = {OptionMetadataTag.DEPRECATED},
- help = "this flag has no effect."
- )
- public boolean dumpActionGraph;
-
- @Deprecated
- @Option(
- name = "dump_action_graph_for_package",
- allowMultiple = true,
- defaultValue = "",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {OptionEffectTag.NO_OP},
- metadataTags = {OptionMetadataTag.DEPRECATED},
- help = "this flag has no effect."
- )
- public List<String> dumpActionGraphForPackage = new ArrayList<>();
-
- @Deprecated
- @Option(
- name = "dump_action_graph_with_middlemen",
- defaultValue = "true",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {OptionEffectTag.NO_OP},
- metadataTags = {OptionMetadataTag.DEPRECATED},
- help = "this flag has no effect."
- )
- public boolean dumpActionGraphWithMiddlemen;
-
- @Deprecated
- @Option(
- name = "dump_providers",
- defaultValue = "false",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {OptionEffectTag.NO_OP},
- metadataTags = {OptionMetadataTag.DEPRECATED},
- help = "This is a no-op."
- )
- public boolean dumpProviders;
-
- @Deprecated
- @Option(
- name = "dump_targets",
- defaultValue = "null",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {OptionEffectTag.NO_OP},
- metadataTags = {OptionMetadataTag.DEPRECATED},
- help = "this flag has no effect."
- )
- public String dumpTargets;
-
- @Deprecated
- @Option(
- name = "dump_host_deps",
- defaultValue = "true",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {OptionEffectTag.NO_OP},
- metadataTags = {OptionMetadataTag.DEPRECATED},
- help = "Deprecated"
- )
- public boolean dumpHostDeps;
-
- @Deprecated
- @Option(
- name = "dump_to_stdout",
- defaultValue = "false",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {OptionEffectTag.NO_OP},
- metadataTags = {OptionMetadataTag.DEPRECATED},
- help = "Deprecated"
- )
- public boolean dumpToStdout;
-
- @Option(
- name = "experimental_post_build_query",
- defaultValue = "null",
- documentationCategory = OptionDocumentationCategory.LOGGING,
- effectTags = {OptionEffectTag.UNKNOWN}
- )
- public String queryExpression;
-
- @Option(
- name = "analyze",
- defaultValue = "true",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {
- OptionEffectTag.LOADING_AND_ANALYSIS,
- OptionEffectTag.AFFECTS_OUTPUTS
- },
- help =
- "Execute the analysis phase; this is the usual behaviour. Specifying --noanalyze causes "
- + "the build to stop before starting the analysis phase, returning zero iff the "
- + "package loading completed successfully; this mode is useful for testing."
- )
- public boolean performAnalysisPhase;
-
- @Option(
- name = "build",
- defaultValue = "true",
- category = "what",
- documentationCategory = OptionDocumentationCategory.OUTPUT_SELECTION,
- effectTags = {
- OptionEffectTag.EXECUTION,
- OptionEffectTag.AFFECTS_OUTPUTS
- },
- help =
- "Execute the build; this is the usual behaviour. "
- + "Specifying --nobuild causes the build to stop before executing the build "
- + "actions, returning zero iff the package loading and analysis phases completed "
- + "successfully; this mode is useful for testing those phases."
- )
- public boolean performExecutionPhase;
-
- @Option(
- name = "output_groups",
- converter = Converters.CommaSeparatedOptionListConverter.class,
- allowMultiple = true,
- documentationCategory = OptionDocumentationCategory.OUTPUT_SELECTION,
- effectTags = {OptionEffectTag.EXECUTION, OptionEffectTag.AFFECTS_OUTPUTS},
- defaultValue = "",
- help =
- "Specifies which output groups of the top-level targets to build. If omitted, a default "
- + "set of output groups are built. When specified the default set is overridden."
- + "However you may use --output_groups=+<output_group> or "
- + "--output_groups=-<output_group> to instead modify the set of output groups."
- )
- public List<String> outputGroups;
-
- @Option(
- name = "show_result",
- defaultValue = "1",
- category = "verbosity",
- documentationCategory = OptionDocumentationCategory.LOGGING,
- effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
- help =
- "Show the results of the build. For each target, state whether or not it was brought "
- + "up-to-date, and if so, a list of output files that were built. The printed files "
- + "are convenient strings for copy+pasting to the shell, to execute them.\n"
- + "This option requires an integer argument, which is the threshold number of "
- + "targets above which result information is not printed. Thus zero causes "
- + "suppression of the message and MAX_INT causes printing of the result to occur "
- + "always. The default is one."
- )
- public int maxResultTargets;
-
- @Option(
- name = "experimental_show_artifacts",
- defaultValue = "false",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
- help =
- "Output a list of all top level artifacts produced by this build."
- + "Use output format suitable for tool consumption. "
- + "This flag is temporary and intended to facilitate Android Studio integration. "
- + "This output format will likely change in the future or disappear completely."
- )
- public boolean showArtifacts;
-
- @Option(
- name = "announce",
- defaultValue = "false",
- category = "verbosity",
- documentationCategory = OptionDocumentationCategory.LOGGING,
- effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
- help = "Deprecated. No-op.",
- deprecationWarning = "This option is now deprecated and is a no-op"
- )
- public boolean announce;
-
- @Option(
- name = "symlink_prefix",
- defaultValue = "null",
- category = "misc",
- documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
- effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
- help =
- "The prefix that is prepended to any of the convenience symlinks that are created "
- + "after a build. If '/' is passed, then no symlinks are created and no warning is "
- + "emitted. If omitted, the default value is the name of the build tool."
- )
- public String symlinkPrefix;
-
- @Option(
- name = "experimental_multi_cpu",
- converter = Converters.CommaSeparatedOptionListConverter.class,
- allowMultiple = true,
- defaultValue = "",
- category = "semantics",
- documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
- effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
- metadataTags = {OptionMetadataTag.EXPERIMENTAL},
- help =
- "This flag allows specifying multiple target CPUs. If this is specified, "
- + "the --cpu option is ignored."
- )
- public List<String> multiCpus;
-
- @Option(
- name = "output_tree_tracking",
- oldName = "experimental_output_tree_tracking",
- defaultValue = "true",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION},
- help =
- "If set, tell the output service (if any) to track when files in the output "
- + "tree have been modified externally (not by the build system). "
- + "This should improve incremental build speed when an appropriate output service "
- + "is enabled."
- )
- public boolean finalizeActions;
-
- @Option(
- name = "aspects",
- converter = Converters.CommaSeparatedOptionListConverter.class,
- defaultValue = "",
- documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
- effectTags = {OptionEffectTag.UNKNOWN},
- allowMultiple = true,
- help =
- "Comma-separated list of aspects to be applied to top-level targets. All aspects "
- + "are applied to all top-level targets independently. Aspects are specified in "
- + "the form <bzl-file-label>%<aspect_name>, "
- + "for example '//tools:my_def.bzl%my_aspect', where 'my_aspect' is a top-level "
- + "value from from a file tools/my_def.bzl"
- )
- public List<String> aspects;
-
- public String getSymlinkPrefix(String productName) {
- return symlinkPrefix == null ? productName + "-" : symlinkPrefix;
- }
-
- // Transitional flag for safely rolling out new convenience symlink behavior.
- // To be made a no-op and deleted once new symlink behavior is battle-tested.
- @Option(
- name = "use_top_level_targets_for_symlinks",
- defaultValue = "false",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
- help =
- "If enabled, the symlinks are based on the configurations of the top-level targets "
- + " rather than the top-level target configuration. If this would be ambiguous, "
- + " the symlinks will be deleted to avoid confusion."
- )
- public boolean useTopLevelTargetsForSymlinks;
-
- /**
- * Returns whether to use the output directories used by the top-level targets for convenience
- * symlinks.
- *
- * <p>If true, then symlinks use the actual output directories of the top-level targets.
- * The symlinks will be created iff all top-level targets share the same output directory.
- * Otherwise, any stale symlinks from previous invocations will be deleted to avoid ambiguity.
- *
- * <p>If false, then symlinks use the output directory implied by command-line flags, regardless
- * of whether top-level targets have transitions which change them (or even have any output
- * directories at all, as in the case of a build with no targets or one which only builds source
- * files).
- */
- public boolean useTopLevelTargetsForSymlinks() {
- return useTopLevelTargetsForSymlinks;
- }
-
- @Option(
- name = "use_action_cache",
- defaultValue = "true",
- documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
- effectTags = {
- OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION,
- OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS
- },
- help = "Whether to use the action cache"
- )
- public boolean useActionCache;
- }
-
- /** Converter for jobs: [0, MAX_JOBS] or "auto". */
- public static class JobsConverter extends RangeConverter {
- /** If not null, indicates the value to return when "auto" is selected. Useful for cases
- * where the number of jobs is bound by another factor different than what we compute here. */
- private static Integer fixedAutoJobs;
-
- public JobsConverter() {
- super(0, MAX_JOBS);
- }
-
- @Override
- public Integer convert(String input) throws OptionsParsingException {
- if (input.equals("auto")) {
- int jobs;
- if (fixedAutoJobs == null) {
- jobs = (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage());
- if (jobs > MAX_JOBS) {
- logger.warning(
- "Detected "
- + jobs
- + " processors, which exceed the maximum allowed number of jobs of "
- + MAX_JOBS
- + "; something seems wrong");
- jobs = MAX_JOBS;
- }
- } else {
- jobs = fixedAutoJobs;
- }
- logger.info("Flag \"jobs\" was set to \"auto\"; using " + jobs + " jobs");
- return jobs;
- } else {
- return super.convert(input);
- }
- }
-
- @Override
- public String getTypeDescription() {
- return "\"auto\" or " + super.getTypeDescription();
- }
-
- /**
- * Sets the value to return by this converter when "auto" is selected.
- *
- * @param jobs the number of jobs to return, or null to reenable automated detection
- */
- public static void setFixedAutoJobs(Integer jobs) {
- Preconditions.checkArgument(jobs == null || jobs <= MAX_JOBS);
- fixedAutoJobs = jobs;
- }
- }
-
- /**
- * Converter for progress_report_interval: [0, 3600].
- */
- public static class ProgressReportIntervalConverter extends RangeConverter {
- public ProgressReportIntervalConverter() {
- super(0, 3600);
- }
- }
-
- @VisibleForTesting public static final int MAX_JOBS = 3000;
- private static final int JOBS_TOO_HIGH_WARNING = 1500;
-
private final UUID id;
private final LoadingCache<Class<? extends OptionsBase>, Optional<OptionsBase>> optionsCache;
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java
new file mode 100644
index 0000000..a771237
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java
@@ -0,0 +1,471 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.buildtool;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.LocalHostCapacity;
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Converters.RangeConverter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionDocumentationCategory;
+import com.google.devtools.common.options.OptionEffectTag;
+import com.google.devtools.common.options.OptionMetadataTag;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParsingException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+
+/**
+ * Options interface for {@link BuildRequest}: can be used to parse command-line arguments.
+ *
+ * <p>See also {@code ExecutionOptions}; from the user's point of view, there's no qualitative
+ * difference between these two sets of options.
+ */
+public class BuildRequestOptions extends OptionsBase {
+ private static final Logger logger = Logger.getLogger(BuildRequestOptions.class.getName());
+ private static final int JOBS_TOO_HIGH_WARNING = 1500;
+ @VisibleForTesting public static final int MAX_JOBS = 3000;
+
+ /* "Execution": options related to the execution of a build: */
+
+ @Option(
+ name = "jobs",
+ abbrev = 'j',
+ defaultValue = "auto",
+ category = "strategy",
+ documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY,
+ effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS, OptionEffectTag.EXECUTION},
+ converter = JobsConverter.class,
+ help =
+ "The number of concurrent jobs to run. 0 means build sequentially."
+ + " \"auto\" means to use a reasonable value derived from the machine's hardware"
+ + " profile (e.g. the number of processors). Values above "
+ + MAX_JOBS
+ + " are not allowed, and values above "
+ + JOBS_TOO_HIGH_WARNING
+ + " may cause memory issues."
+ )
+ public int jobs;
+
+ @Option(
+ name = "progress_report_interval",
+ defaultValue = "0",
+ category = "verbosity",
+ documentationCategory = OptionDocumentationCategory.LOGGING,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ converter = ProgressReportIntervalConverter.class,
+ help =
+ "The number of seconds to wait between two reports on still running jobs. The "
+ + "default value 0 means to use the default 10:30:60 incremental algorithm."
+ )
+ public int progressReportInterval;
+
+ @Option(
+ name = "explain",
+ defaultValue = "null",
+ category = "verbosity",
+ documentationCategory = OptionDocumentationCategory.LOGGING,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help =
+ "Causes the build system to explain each executed step of the "
+ + "build. The explanation is written to the specified log file."
+ )
+ public PathFragment explanationPath;
+
+ @Option(
+ name = "verbose_explanations",
+ defaultValue = "false",
+ category = "verbosity",
+ documentationCategory = OptionDocumentationCategory.LOGGING,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "Increases the verbosity of the explanations issued if --explain is enabled. "
+ + "Has no effect if --explain is not enabled."
+ )
+ public boolean verboseExplanations;
+
+ @Option(
+ name = "output_filter",
+ converter = Converters.RegexPatternConverter.class,
+ defaultValue = "null",
+ category = "flags",
+ documentationCategory = OptionDocumentationCategory.LOGGING,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help = "Only shows warnings for rules with a name matching the provided regular expression."
+ )
+ public Pattern outputFilter;
+
+ @Deprecated
+ @Option(
+ name = "dump_makefile",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.NO_OP},
+ metadataTags = {OptionMetadataTag.DEPRECATED},
+ help = "this flag has no effect."
+ )
+ public boolean dumpMakefile;
+
+ @Deprecated
+ @Option(
+ name = "dump_action_graph",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.NO_OP},
+ metadataTags = {OptionMetadataTag.DEPRECATED},
+ help = "this flag has no effect."
+ )
+ public boolean dumpActionGraph;
+
+ @Deprecated
+ @Option(
+ name = "dump_action_graph_for_package",
+ allowMultiple = true,
+ defaultValue = "",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.NO_OP},
+ metadataTags = {OptionMetadataTag.DEPRECATED},
+ help = "this flag has no effect."
+ )
+ public List<String> dumpActionGraphForPackage = new ArrayList<>();
+
+ @Deprecated
+ @Option(
+ name = "dump_action_graph_with_middlemen",
+ defaultValue = "true",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.NO_OP},
+ metadataTags = {OptionMetadataTag.DEPRECATED},
+ help = "this flag has no effect."
+ )
+ public boolean dumpActionGraphWithMiddlemen;
+
+ @Deprecated
+ @Option(
+ name = "dump_providers",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.NO_OP},
+ metadataTags = {OptionMetadataTag.DEPRECATED},
+ help = "This is a no-op."
+ )
+ public boolean dumpProviders;
+
+ @Deprecated
+ @Option(
+ name = "dump_targets",
+ defaultValue = "null",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.NO_OP},
+ metadataTags = {OptionMetadataTag.DEPRECATED},
+ help = "this flag has no effect."
+ )
+ public String dumpTargets;
+
+ @Deprecated
+ @Option(
+ name = "dump_host_deps",
+ defaultValue = "true",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.NO_OP},
+ metadataTags = {OptionMetadataTag.DEPRECATED},
+ help = "Deprecated"
+ )
+ public boolean dumpHostDeps;
+
+ @Deprecated
+ @Option(
+ name = "dump_to_stdout",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.NO_OP},
+ metadataTags = {OptionMetadataTag.DEPRECATED},
+ help = "Deprecated"
+ )
+ public boolean dumpToStdout;
+
+ @Option(
+ name = "experimental_post_build_query",
+ defaultValue = "null",
+ documentationCategory = OptionDocumentationCategory.LOGGING,
+ effectTags = {OptionEffectTag.UNKNOWN}
+ )
+ public String queryExpression;
+
+ @Option(
+ name = "analyze",
+ defaultValue = "true",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS, OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "Execute the analysis phase; this is the usual behaviour. Specifying --noanalyze causes "
+ + "the build to stop before starting the analysis phase, returning zero iff the "
+ + "package loading completed successfully; this mode is useful for testing."
+ )
+ public boolean performAnalysisPhase;
+
+ @Option(
+ name = "build",
+ defaultValue = "true",
+ category = "what",
+ documentationCategory = OptionDocumentationCategory.OUTPUT_SELECTION,
+ effectTags = {OptionEffectTag.EXECUTION, OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "Execute the build; this is the usual behaviour. "
+ + "Specifying --nobuild causes the build to stop before executing the build "
+ + "actions, returning zero iff the package loading and analysis phases completed "
+ + "successfully; this mode is useful for testing those phases."
+ )
+ public boolean performExecutionPhase;
+
+ @Option(
+ name = "output_groups",
+ converter = Converters.CommaSeparatedOptionListConverter.class,
+ allowMultiple = true,
+ documentationCategory = OptionDocumentationCategory.OUTPUT_SELECTION,
+ effectTags = {OptionEffectTag.EXECUTION, OptionEffectTag.AFFECTS_OUTPUTS},
+ defaultValue = "",
+ help =
+ "Specifies which output groups of the top-level targets to build. If omitted, a default "
+ + "set of output groups are built. When specified the default set is overridden."
+ + "However you may use --output_groups=+<output_group> or "
+ + "--output_groups=-<output_group> to instead modify the set of output groups."
+ )
+ public List<String> outputGroups;
+
+ @Option(
+ name = "show_result",
+ defaultValue = "1",
+ category = "verbosity",
+ documentationCategory = OptionDocumentationCategory.LOGGING,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "Show the results of the build. For each target, state whether or not it was brought "
+ + "up-to-date, and if so, a list of output files that were built. The printed files "
+ + "are convenient strings for copy+pasting to the shell, to execute them.\n"
+ + "This option requires an integer argument, which is the threshold number of "
+ + "targets above which result information is not printed. Thus zero causes "
+ + "suppression of the message and MAX_INT causes printing of the result to occur "
+ + "always. The default is one."
+ )
+ public int maxResultTargets;
+
+ @Option(
+ name = "experimental_show_artifacts",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "Output a list of all top level artifacts produced by this build."
+ + "Use output format suitable for tool consumption. "
+ + "This flag is temporary and intended to facilitate Android Studio integration. "
+ + "This output format will likely change in the future or disappear completely."
+ )
+ public boolean showArtifacts;
+
+ @Option(
+ name = "announce",
+ defaultValue = "false",
+ category = "verbosity",
+ documentationCategory = OptionDocumentationCategory.LOGGING,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help = "Deprecated. No-op.",
+ deprecationWarning = "This option is now deprecated and is a no-op"
+ )
+ public boolean announce;
+
+ @Option(
+ name = "symlink_prefix",
+ defaultValue = "null",
+ category = "misc",
+ documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "The prefix that is prepended to any of the convenience symlinks that are created "
+ + "after a build. If '/' is passed, then no symlinks are created and no warning is "
+ + "emitted. If omitted, the default value is the name of the build tool."
+ )
+ public String symlinkPrefix;
+
+ @Option(
+ name = "experimental_multi_cpu",
+ converter = Converters.CommaSeparatedOptionListConverter.class,
+ allowMultiple = true,
+ defaultValue = "",
+ category = "semantics",
+ documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ metadataTags = {OptionMetadataTag.EXPERIMENTAL},
+ help =
+ "This flag allows specifying multiple target CPUs. If this is specified, "
+ + "the --cpu option is ignored."
+ )
+ public List<String> multiCpus;
+
+ @Option(
+ name = "output_tree_tracking",
+ oldName = "experimental_output_tree_tracking",
+ defaultValue = "true",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION},
+ help =
+ "If set, tell the output service (if any) to track when files in the output "
+ + "tree have been modified externally (not by the build system). "
+ + "This should improve incremental build speed when an appropriate output service "
+ + "is enabled."
+ )
+ public boolean finalizeActions;
+
+ @Option(
+ name = "aspects",
+ converter = Converters.CommaSeparatedOptionListConverter.class,
+ defaultValue = "",
+ documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ allowMultiple = true,
+ help =
+ "Comma-separated list of aspects to be applied to top-level targets. All aspects "
+ + "are applied to all top-level targets independently. Aspects are specified in "
+ + "the form <bzl-file-label>%<aspect_name>, "
+ + "for example '//tools:my_def.bzl%my_aspect', where 'my_aspect' is a top-level "
+ + "value from from a file tools/my_def.bzl"
+ )
+ public List<String> aspects;
+
+ public String getSymlinkPrefix(String productName) {
+ return symlinkPrefix == null ? productName + "-" : symlinkPrefix;
+ }
+
+ // Transitional flag for safely rolling out new convenience symlink behavior.
+ // To be made a no-op and deleted once new symlink behavior is battle-tested.
+ @Option(
+ name = "use_top_level_targets_for_symlinks",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "If enabled, the symlinks are based on the configurations of the top-level targets "
+ + " rather than the top-level target configuration. If this would be ambiguous, "
+ + " the symlinks will be deleted to avoid confusion."
+ )
+ public boolean useTopLevelTargetsForSymlinks;
+
+ /**
+ * Returns whether to use the output directories used by the top-level targets for convenience
+ * symlinks.
+ *
+ * <p>If true, then symlinks use the actual output directories of the top-level targets. The
+ * symlinks will be created iff all top-level targets share the same output directory. Otherwise,
+ * any stale symlinks from previous invocations will be deleted to avoid ambiguity.
+ *
+ * <p>If false, then symlinks use the output directory implied by command-line flags, regardless
+ * of whether top-level targets have transitions which change them (or even have any output
+ * directories at all, as in the case of a build with no targets or one which only builds source
+ * files).
+ */
+ public boolean useTopLevelTargetsForSymlinks() {
+ return useTopLevelTargetsForSymlinks;
+ }
+
+ @Option(
+ name = "use_action_cache",
+ defaultValue = "true",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {
+ OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION,
+ OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS
+ },
+ help = "Whether to use the action cache"
+ )
+ public boolean useActionCache;
+
+ @Option(
+ name = "keep_incrementality_data",
+ defaultValue = "true",
+ documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION,
+ effectTags = {OptionEffectTag.LOSES_INCREMENTAL_STATE},
+ help =
+ "If false, discard Blaze-internal data that allows for invalidation and re-evaluation "
+ + "on incremental builds in order to save memory on this build. Subsequent builds "
+ + "will not have any incrementality with respect to this one. Usually you will want"
+ + "to specify the --batch startup option along with this one."
+ )
+ public boolean keepIncrementalityData;
+
+ /** Converter for jobs: [0, MAX_JOBS] or "auto". */
+ public static class JobsConverter extends RangeConverter {
+ /**
+ * If not null, indicates the value to return when "auto" is selected. Useful for cases where
+ * the number of jobs is bound by another factor different than what we compute here.
+ */
+ private static Integer fixedAutoJobs;
+
+ public JobsConverter() {
+ super(0, MAX_JOBS);
+ }
+
+ @Override
+ public Integer convert(String input) throws OptionsParsingException {
+ if (input.equals("auto")) {
+ int jobs;
+ if (fixedAutoJobs == null) {
+ jobs = (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage());
+ if (jobs > MAX_JOBS) {
+ logger.warning(
+ "Detected "
+ + jobs
+ + " processors, which exceed the maximum allowed number of jobs of "
+ + MAX_JOBS
+ + "; something seems wrong");
+ jobs = MAX_JOBS;
+ }
+ } else {
+ jobs = fixedAutoJobs;
+ }
+ logger.info("Flag \"jobs\" was set to \"auto\"; using " + jobs + " jobs");
+ return jobs;
+ } else {
+ return super.convert(input);
+ }
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "\"auto\" or " + super.getTypeDescription();
+ }
+
+ /**
+ * Sets the value to return by this converter when "auto" is selected.
+ *
+ * @param jobs the number of jobs to return, or null to reenable automated detection
+ */
+ public static void setFixedAutoJobs(Integer jobs) {
+ Preconditions.checkArgument(jobs == null || jobs <= MAX_JOBS);
+ fixedAutoJobs = jobs;
+ }
+ }
+
+ /** Converter for progress_report_interval: [0, 3600]. */
+ public static class ProgressReportIntervalConverter extends RangeConverter {
+ public ProgressReportIntervalConverter() {
+ super(0, 3600);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
index 7290852..d785f51 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
@@ -40,7 +40,6 @@
import com.google.devtools.build.lib.buildeventstream.AbortedEvent;
import com.google.devtools.build.lib.buildeventstream.BuildEventId;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.Aborted.AbortReason;
-import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions;
import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent;
import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent;
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
index 24d859a..e878b6f 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -437,10 +437,9 @@
executor.executionPhaseStarting();
skyframeExecutor.drainChangedFiles();
- if (request.getViewOptions().discardAnalysisCache) {
- // Free memory by removing cache entries that aren't going to be needed. Note that in
- // skyframe full, this destroys the action graph as well, so we can only do it after the
- // action graph is no longer needed.
+ if (request.getViewOptions().discardAnalysisCache
+ || !request.getBuildOptions().keepIncrementalityData) {
+ // Free memory by removing cache entries that aren't going to be needed.
env.getSkyframeBuildView()
.clearAnalysisCache(analysisResult.getTargetsToBuild(), analysisResult.getAspects());
}
@@ -685,7 +684,7 @@
ActionCache actionCache,
SkyframeExecutor skyframeExecutor,
ModifiedFileSet modifiedOutputFiles) {
- BuildRequest.BuildRequestOptions options = request.getBuildOptions();
+ BuildRequestOptions options = request.getBuildOptions();
boolean keepGoing = request.getViewOptions().keepGoing;
Path actionOutputRoot = env.getActionConsoleOutputDirectory();
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java
index d8e5a51..54552a6 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java
@@ -21,7 +21,6 @@
import com.google.devtools.build.lib.actions.PackageRootResolver;
import com.google.devtools.build.lib.actions.cache.ActionCache;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
-import com.google.devtools.build.lib.analysis.BuildView;
import com.google.devtools.build.lib.analysis.SkyframePackageRootResolver;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.DefaultsPackage;
@@ -607,9 +606,10 @@
// Fail fast in the case where a Blaze command forgets to install the package path correctly.
skyframeExecutor.setActive(false);
// Let skyframe figure out if it needs to store graph edges for this build.
- skyframeExecutor.decideKeepIncrementalStateAndResetEvaluatorIfNecessary(
+ skyframeExecutor.decideKeepIncrementalState(
runtime.getStartupOptionsProvider().getOptions(BlazeServerStartupOptions.class).batch,
- options.getOptions(BuildView.Options.class));
+ options,
+ reporter);
// Start the performance and memory profilers.
runtime.beforeCommand(this, commonOptions, execStartTimeNanos);
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java
index 5f8077f..d0aa57b 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java
@@ -15,7 +15,7 @@
import com.google.devtools.build.lib.analysis.BuildView;
import com.google.devtools.build.lib.buildtool.BuildRequest;
-import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions;
+import com.google.devtools.build.lib.buildtool.BuildRequestOptions;
import com.google.devtools.build.lib.buildtool.BuildTool;
import com.google.devtools.build.lib.exec.ExecutionOptions;
import com.google.devtools.build.lib.exec.local.LocalExecutionOptions;
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java
index 88ff555..ba0ca59 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java
@@ -15,7 +15,7 @@
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.analysis.NoBuildEvent;
-import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.BuildRequestOptions;
import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.runtime.BlazeCommand;
@@ -182,8 +182,10 @@
env.getReporter().handle(Event.info(null/*location*/, cleanBanner));
try {
- String symlinkPrefix = options.getOptions(BuildRequest.BuildRequestOptions.class)
- .getSymlinkPrefix(env.getRuntime().getProductName());
+ String symlinkPrefix =
+ options
+ .getOptions(BuildRequestOptions.class)
+ .getSymlinkPrefix(env.getRuntime().getProductName());
actuallyClean(env, env.getOutputBase(), expunge, expungeAsync, async, symlinkPrefix);
return ExitCode.SUCCESS;
} catch (IOException e) {
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java
index 17f204a..aee56d4 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java
@@ -28,7 +28,7 @@
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.RunUnder;
import com.google.devtools.build.lib.buildtool.BuildRequest;
-import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions;
+import com.google.devtools.build.lib.buildtool.BuildRequestOptions;
import com.google.devtools.build.lib.buildtool.BuildResult;
import com.google.devtools.build.lib.buildtool.BuildTool;
import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils;
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
index cf3c2cf..5dac255 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
@@ -27,13 +27,16 @@
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
-import com.google.devtools.build.lib.analysis.BuildView.Options;
+import com.google.devtools.build.lib.analysis.BuildView;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
+import com.google.devtools.build.lib.buildtool.BuildRequestOptions;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.concurrent.Uninterruptibles;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.packages.AspectClass;
import com.google.devtools.build.lib.packages.Package;
@@ -77,6 +80,7 @@
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.common.options.OptionsClassProvider;
+import com.google.devtools.common.options.OptionsProvider;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collection;
@@ -107,13 +111,19 @@
CLEAR_EDGES_AND_ACTIONS
}
- // Can only be set once over the lifetime of this object. If CLEAR_EDGES or
- // CLEAR_EDGES_AND_ACTIONS, the graph will not store edges, saving memory but making incremental
- // builds impossible. If CLEAR_EDGES_AND_ACTIONS, each action will be dereferenced once it is
- // executed, saving memory.
+ /**
+ * If {@link IncrementalState#CLEAR_EDGES_AND_ACTIONS}, the graph will not store edges, saving
+ * memory but making subsequent builds not incremental. Also, each action will be dereferenced
+ * once it is executed, saving memory.
+ */
private IncrementalState incrementalState = IncrementalState.NORMAL;
- private RecordingDifferencer recordingDiffer;
+ private boolean evaluatorNeedsReset = false;
+
+ // This is intentionally not kept in sync with the evaluator: we may reset the evaluator without
+ // ever losing injected/invalidated data here. This is safe because the worst that will happen is
+ // that on the next build we try to inject/invalidate some nodes that aren't needed for the build.
+ private final RecordingDifferencer recordingDiffer = new SequencedRecordingDifferencer();
private final DiffAwarenessManager diffAwarenessManager;
private final Iterable<SkyValueDirtinessChecker> customDirtinessCheckers;
private Set<String> previousClientEnvironment = null;
@@ -191,14 +201,6 @@
}
@Override
- protected void init() {
- // Note that we need to set recordingDiffer first since SkyframeExecutor#init calls
- // SkyframeExecutor#evaluatorDiffer.
- recordingDiffer = new SequencedRecordingDifferencer();
- super.init();
- }
-
- @Override
public void resetEvaluator() {
super.resetEvaluator();
diffAwarenessManager.reset();
@@ -232,6 +234,12 @@
TimestampGranularityMonitor tsgm,
OptionsClassProvider options)
throws InterruptedException, AbruptExitException {
+ if (evaluatorNeedsReset) {
+ // Recreate MemoizingEvaluator so that graph is recreated with correct edge-clearing status,
+ // or if the graph doesn't have edges, so that a fresh graph can be used.
+ resetEvaluator();
+ evaluatorNeedsReset = false;
+ }
super.sync(eventHandler, packageCacheOptions, skylarkSemanticsOptions, outputBase,
workingDirectory, defaultsPackageContents, commandId, clientEnv, tsgm, options);
handleDiffs(eventHandler, packageCacheOptions.checkOutputFiles, options);
@@ -490,24 +498,44 @@
}
@Override
- public void decideKeepIncrementalStateAndResetEvaluatorIfNecessary(
- boolean batch, Options viewOptions) {
+ public void decideKeepIncrementalState(
+ boolean batch, OptionsProvider options, EventHandler eventHandler) {
Preconditions.checkState(!active);
- if (viewOptions == null) {
- // Some blaze commands don't include the view options. Don't bother with them.
- return;
+ BuildView.Options viewOptions = options.getOptions(BuildView.Options.class);
+ BuildRequestOptions requestOptions = options.getOptions(BuildRequestOptions.class);
+ boolean explicitlyRequestedNoIncrementalData =
+ requestOptions != null && !requestOptions.keepIncrementalityData;
+ boolean implicitlyRequestedNoIncrementalData =
+ batch && viewOptions != null && viewOptions.discardAnalysisCache;
+ boolean discardingEdges =
+ explicitlyRequestedNoIncrementalData || implicitlyRequestedNoIncrementalData;
+ if (explicitlyRequestedNoIncrementalData != implicitlyRequestedNoIncrementalData) {
+ if (requestOptions != null && !explicitlyRequestedNoIncrementalData) {
+ eventHandler.handle(
+ Event.warn(
+ "--batch and --discard_analysis_cache specified, but --nokeep_incrementality_data "
+ + "not specified: incrementality data is implicitly discarded, but you may need"
+ + " to specify --nokeep_incrementality_data in the future if you want to "
+ + "maximize memory savings."));
+ }
+ if (!batch) {
+ eventHandler.handle(
+ Event.warn(
+ "--batch not specified with --nokeep_incrementality_data: the server will "
+ + "remain running, but the next build will not be incremental on this one."));
+ }
}
- if (batch && viewOptions.discardAnalysisCache) {
- Preconditions.checkState(
- incrementalState == IncrementalState.NORMAL,
- "May only be called once if successful: %s",
- incrementalState);
- incrementalState = IncrementalState.CLEAR_EDGES_AND_ACTIONS;
+ IncrementalState oldState = incrementalState;
+ incrementalState =
+ discardingEdges ? IncrementalState.CLEAR_EDGES_AND_ACTIONS : IncrementalState.NORMAL;
+ if (oldState != incrementalState) {
logger.info("Set incremental state to " + incrementalState);
- // Recreate MemoizingEvaluator so that graph is recreated with edge-clearing enabled.
- resetEvaluator();
+ evaluatorNeedsReset = true;
+ removeActionsAfterEvaluation.set(
+ incrementalState == IncrementalState.CLEAR_EDGES_AND_ACTIONS);
+ } else if (incrementalState == IncrementalState.CLEAR_EDGES_AND_ACTIONS) {
+ evaluatorNeedsReset = true;
}
- removeActionsAfterEvaluation.set(incrementalState == IncrementalState.CLEAR_EDGES_AND_ACTIONS);
}
@Override
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
index 413ea01..ec36566 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -53,7 +53,6 @@
import com.google.devtools.build.lib.actions.Root;
import com.google.devtools.build.lib.analysis.AspectCollection;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
-import com.google.devtools.build.lib.analysis.BuildView.Options;
import com.google.devtools.build.lib.analysis.ConfiguredAspect;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
@@ -81,6 +80,7 @@
import com.google.devtools.build.lib.concurrent.ThreadSafety;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.events.ErrorSensingEventHandler;
+import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.packages.AspectDescriptor;
@@ -149,6 +149,7 @@
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.WalkableGraph.WalkableGraphFactory;
import com.google.devtools.common.options.OptionsClassProvider;
+import com.google.devtools.common.options.OptionsProvider;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collection;
@@ -682,8 +683,8 @@
}
/**
- * Decides if graph edges should be stored for this build. If not, re-creates the graph to not
- * store graph edges. Necessary conditions to not store graph edges are:
+ * Decides if graph edges should be stored for this build. If not, notes that the next evaluation
+ * on the graph should reset it first. Necessary conditions to not store graph edges are:
*
* <ol>
* <li>batch (since incremental builds are not possible);
@@ -692,8 +693,8 @@
* way).
* </ol>
*/
- public void decideKeepIncrementalStateAndResetEvaluatorIfNecessary(
- boolean batch, Options viewOptions) {
+ public void decideKeepIncrementalState(
+ boolean batch, OptionsProvider viewOptions, EventHandler eventHandler) {
// Assume incrementality.
}
diff --git a/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java
index ec636b3..541d340 100644
--- a/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java
+++ b/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java
@@ -349,11 +349,15 @@
if (entry.isDone()) {
out.print(keyFormatter.apply(key));
out.print("|");
- try {
- out.println(
- Joiner.on('|').join(Iterables.transform(entry.getDirectDeps(), keyFormatter)));
- } catch (InterruptedException e) {
- throw new IllegalStateException("InMemoryGraph doesn't throw: " + entry, e);
+ if (((InMemoryNodeEntry) entry).keepEdges() == NodeEntry.KeepEdgesPolicy.NONE) {
+ out.println(" (direct deps not stored)");
+ } else {
+ try {
+ out.println(
+ Joiner.on('|').join(Iterables.transform(entry.getDirectDeps(), keyFormatter)));
+ } catch (InterruptedException e) {
+ throw new IllegalStateException("InMemoryGraph doesn't throw: " + entry, e);
+ }
}
}
}
diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index 06bc832..a9fc419 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -366,6 +366,7 @@
"//src/main/java/com/google/devtools/build/lib:bazel-main",
"//src/main/java/com/google/devtools/build/lib:bazel-rules",
"//src/main/java/com/google/devtools/build/lib:build-base",
+ "//src/main/java/com/google/devtools/build/lib:build-request-options",
"//src/main/java/com/google/devtools/build/lib:core-rules",
"//src/main/java/com/google/devtools/build/lib:events",
"//src/main/java/com/google/devtools/build/lib:io",
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java
index 5786637..16c991b 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java
@@ -36,7 +36,7 @@
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
import com.google.devtools.build.lib.analysis.configuredtargets.InputFileConfiguredTarget;
-import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions;
+import com.google.devtools.build.lib.buildtool.BuildRequestOptions;
import com.google.devtools.build.lib.clock.BlazeClock;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
index 987dce9..1a5b3dc 100644
--- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
+++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java
@@ -83,7 +83,7 @@
import com.google.devtools.build.lib.analysis.extra.ExtraAction;
import com.google.devtools.build.lib.analysis.test.BaselineCoverageAction;
import com.google.devtools.build.lib.analysis.test.InstrumentedFilesProvider;
-import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.BuildRequestOptions;
import com.google.devtools.build.lib.clock.BlazeClock;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
@@ -296,10 +296,11 @@
protected final BuildConfigurationCollection createConfigurations(String... args)
throws Exception {
- optionsParser = OptionsParser.newOptionsParser(Iterables.concat(Arrays.asList(
- ExecutionOptions.class,
- BuildRequest.BuildRequestOptions.class),
- ruleClassProvider.getConfigurationOptions()));
+ optionsParser =
+ OptionsParser.newOptionsParser(
+ Iterables.concat(
+ Arrays.asList(ExecutionOptions.class, BuildRequestOptions.class),
+ ruleClassProvider.getConfigurationOptions()));
List<String> allArgs = new ArrayList<>();
// TODO(dmarting): Add --stamp option only to test that requires it.
allArgs.add("--stamp"); // Stamp is now defaulted to false.
diff --git a/src/test/shell/integration/BUILD b/src/test/shell/integration/BUILD
index 1790710..7182dbe 100644
--- a/src/test/shell/integration/BUILD
+++ b/src/test/shell/integration/BUILD
@@ -211,7 +211,7 @@
":discard_graph_edges_lib.sh",
":test-deps",
],
- shard_count = 5,
+ shard_count = 6,
)
sh_test(
diff --git a/src/test/shell/integration/discard_graph_edges_lib.sh b/src/test/shell/integration/discard_graph_edges_lib.sh
index e5bea5b..683b96b 100644
--- a/src/test/shell/integration/discard_graph_edges_lib.sh
+++ b/src/test/shell/integration/discard_graph_edges_lib.sh
@@ -17,7 +17,7 @@
# discard_graph_edges_lib.sh: functions needed by discard_graph_edges_test.sh
STARTUP_FLAGS="--batch"
-BUILD_FLAGS="--discard_analysis_cache"
+BUILD_FLAGS="--discard_analysis_cache --nokeep_incrementality_data"
function extract_histogram_count() {
local histofile="$1"
@@ -64,14 +64,23 @@
local readonly histo_root="$("$product" info \
"${PRODUCT_NAME:-$product}-genfiles" 2> /dev/null)/histodump/histo."
"$product" clean >& "$TEST_log" || fail "Couldn't clean"
+ readonly local explicit_server_pid="$("$product" $STARTUP_FLAGS info \
+ server_pid)"
"$product" $STARTUP_FLAGS build --show_timestamps $BUILD_FLAGS \
$extra_build_arg //histodump:action3 >> "$TEST_log" 2>&1 &
subshell_pid="$!"
cat "$exec_fifo" > /dev/null
- if [[ -z "$get_pid_expression" ]]; then
- server_pid="$subshell_pid"
+ # We plan to remove batch mode from the relevant flags for discarding
+ # incrementality state. In the interim, tests that are not in batch mode
+ # explicitly pass --nobatch, so we can use it as a signal.
+ if [[ "$STARTUP_FLAGS" =~ "--nobatch" ]]; then
+ server_pid="$explicit_server_pid"
else
- server_pid="$($get_pid_expression)"
+ if [[ -z "$get_pid_expression" ]]; then
+ server_pid="$subshell_pid"
+ else
+ server_pid="$($get_pid_expression)"
+ fi
fi
echo "server_pid in main thread is ${server_pid}" # >> "$TEST_log"
echo "$server_pid" > "$server_pid_file"
@@ -96,4 +105,3 @@
genrule_action_count="$new_genrule_action_count"
done
}
-
diff --git a/src/test/shell/integration/discard_graph_edges_test.sh b/src/test/shell/integration/discard_graph_edges_test.sh
index e6b2e4d..4b0311c 100755
--- a/src/test/shell/integration/discard_graph_edges_test.sh
+++ b/src/test/shell/integration/discard_graph_edges_test.sh
@@ -179,15 +179,24 @@
histo_file="$(bazel info "${PRODUCT_NAME}-genfiles" \
2> /dev/null)/histodump/histo.txt"
bazel clean >& "$TEST_log" || fail "Couldn't clean"
+ readonly local explicit_server_pid="$(bazel $STARTUP_FLAGS info server_pid)"
bazel $STARTUP_FLAGS build --show_timestamps $build_args \
//histodump:histodump >> "$TEST_log" 2>&1 &
- server_pid=$!
+ readonly local subshell_pid=$!
+ # We plan to remove batch mode from the relevant flags for discarding
+ # incrementality state. In the interim, tests that are not in batch mode
+ # explicitly pass --nobatch, so we can use it as a signal.
+ if [[ "$STARTUP_FLAGS" =~ "--nobatch" ]]; then
+ server_pid="$explicit_server_pid"
+ else
+ server_pid="$subshell_pid"
+ fi
echo "server_pid in main thread is ${server_pid}" >> "$TEST_log"
echo "$server_pid" > "$server_pid_fifo"
echo "Finished writing pid to fifo at " >> "$TEST_log"
date >> "$TEST_log"
# Wait for previous command to finish.
- wait "$server_pid" || fail "Bazel command failed"
+ wait "$subshell_pid" || fail "Bazel command failed"
cat "$histo_file" >> "$TEST_log"
echo "$histo_file"
}
@@ -325,13 +334,107 @@
# The following tests are not expected to exercise codepath -- make sure nothing bad happens.
function test_no_batch() {
- bazel $STARTUP_FLAGS --nobatch test $BUILD_FLAGS //testing:mytest >& $TEST_log \
- || fail "Expected success"
+ bazel $STARTUP_FLAGS --nobatch test $BUILD_FLAGS --keep_incrementality_data \
+ //testing:mytest >& "$TEST_log" || fail "Expected success"
}
function test_no_discard_analysis_cache() {
- bazel $STARTUP_FLAGS test $BUILD_FLAGS --nodiscard_analysis_cache //testing:mytest >& $TEST_log \
- || fail "Expected success"
+ bazel $STARTUP_FLAGS test $BUILD_FLAGS --nodiscard_analysis_cache \
+ --keep_incrementality_data //testing:mytest >& "$TEST_log" \
+ || fail "Expected success"
+}
+
+function test_packages_cleared_nobatch() {
+ readonly local old_startup_flags="$STARTUP_FLAGS"
+ STARTUP_FLAGS="--nobatch"
+ readonly local old_build_flags="$BUILD_FLAGS"
+ BUILD_FLAGS="--nokeep_incrementality_data --discard_analysis_cache"
+ test_packages_cleared
+ STARTUP_FLAGS="$old_startup_flags"
+ BUILD_FLAGS="$old_build_flags"
+}
+
+function test_packages_cleared_implicit_noincrementality_data() {
+ readonly local old_build_flags="$BUILD_FLAGS"
+ BUILD_FLAGS="$BUILD_FLAGS --keep_incrementality_data"
+ test_packages_cleared
+ BUILD_FLAGS="$old_build_flags"
+}
+
+function test_actions_deleted_after_execution_nobatch_keep_analysis () {
+ readonly local old_startup_flags="$STARTUP_FLAGS"
+ STARTUP_FLAGS="--nobatch"
+ readonly local old_build_flags="$BUILD_FLAGS"
+ BUILD_FLAGS="--nokeep_incrementality_data"
+ test_actions_deleted_after_execution
+ STARTUP_FLAGS="$old_startup_flags"
+ BUILD_FLAGS="$old_build_flags"
+}
+
+function test_dump_after_discard_incrementality_data() {
+ bazel build --nokeep_incrementality_data //testing:mytest >& "$TEST_log" \
+ || fail "Expected success"
+ bazel dump --skyframe=detailed >& "$TEST_log" || fail "Expected success"
+ expect_log "//testing:mytest"
+}
+
+function test_query_after_discard_incrementality_data() {
+ bazel build --nobuild --nokeep_incrementality_data //testing:mytest \
+ >& "$TEST_log" || fail "Expected success"
+ bazel query --noexperimental_ui --output=label_kind //testing:mytest \
+ >& "$TEST_log" || fail "Expected success"
+ expect_log "Loading package: testing"
+ expect_log "cc_test rule //testing:mytest"
+}
+
+function test_shutdown_after_discard_incrementality_data() {
+ readonly local server_pid="$(bazel info server_pid 2> /dev/null)"
+ [[ -z "$server_pid" ]] && fail "Couldn't get server pid"
+ bazel build --nobuild --nokeep_incrementality_data //testing:mytest \
+ >& "$TEST_log" || fail "Expected success"
+ bazel shutdown || fail "Expected success"
+ readonly local new_server_pid="$(bazel info server_pid 2> /dev/null)"
+ [[ "$server_pid" != "$new_server_pid" ]] \
+ || fail "pids $server_pid and $new_server_pid equal"
+}
+
+function test_clean_after_discard_incrementality_data() {
+ bazel build --nobuild --nokeep_incrementality_data //testing:mytest \
+ >& "$TEST_log" || fail "Expected success"
+ bazel clean >& "$TEST_log" || fail "Expected success"
+}
+
+function test_switch_back_and_forth() {
+ readonly local server_pid="$(bazel info \
+ --nokeep_incrementality_data server_pid 2> /dev/null)"
+ [[ -z "$server_pid" ]] && fail "Couldn't get server pid"
+ bazel test --noexperimental_ui --nokeep_incrementality_data \
+ //testing:mytest >& "$TEST_log" || fail "Expected success"
+ expect_log "Loading package: testing"
+ bazel test --noexperimental_ui --nokeep_incrementality_data \
+ //testing:mytest >& "$TEST_log" || fail "Expected success"
+ expect_log "Loading package: testing"
+ bazel test --noexperimental_ui //testing:mytest >& "$TEST_log" \
+ || fail "Expected success"
+ expect_log "Loading package: testing"
+ bazel test --noexperimental_ui //testing:mytest >& "$TEST_log" \
+ || fail "Expected success"
+ expect_not_log "Loading package: testing"
+ bazel test --noexperimental_ui --nokeep_incrementality_data \
+ //testing:mytest >& "$TEST_log" || fail "Expected success"
+ expect_log "Loading package: testing"
+ readonly local new_server_pid="$(bazel info server_pid 2> /dev/null)"
+ [[ "$server_pid" == "$new_server_pid" ]] \
+ || fail "pids $server_pid and $new_server_pid not equal"
+}
+
+function test_warns_on_unexpected_combos() {
+ bazel --batch build --nobuild --discard_analysis_cache >& "$TEST_log" \
+ || fail "Expected success"
+ expect_log "--batch and --discard_analysis_cache specified, but --nokeep_incrementality_data not specified"
+ bazel build --nobuild --discard_analysis_cache --nokeep_incrementality_data \
+ >& "$TEST_log" || fail "Expected success"
+ expect_log "--batch not specified with --nokeep_incrementality_data"
}
run_suite "test for --discard_graph_edges"
