Update from Google.
--
MOE_MIGRATED_REVID=85702957
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java b/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java
new file mode 100644
index 0000000..9bf7a3f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java
@@ -0,0 +1,120 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+
+import javax.annotation.Nullable;
+
+/**
+ * This class records the critical path for the graph of actions executed.
+ */
+@ThreadCompatible
+public class AbstractCriticalPathComponent<C extends AbstractCriticalPathComponent<C>> {
+
+ /** Wall time start time for the action. In milliseconds. */
+ private final long startTime;
+ /** Wall time finish time for the action. In milliseconds. */
+ private long finishTime = 0;
+ protected volatile boolean isRunning = true;
+
+ /** We keep here the critical path time for the most expensive child. */
+ private long childAggregatedWallTime = 0;
+
+ /** The action for which we are storing the stat. */
+ private final Action action;
+
+ /**
+ * Child with the maximum critical path.
+ */
+ @Nullable
+ private C child;
+
+ public AbstractCriticalPathComponent(Action action, long startTime) {
+ this.action = action;
+ this.startTime = startTime;
+ }
+
+ /** Sets the finish time for the action in milliseconds. */
+ public void setFinishTimeMillis(long finishTime) {
+ Preconditions.checkState(isRunning, "Already stopped! %s.", action);
+ this.finishTime = finishTime;
+ isRunning = false;
+ }
+
+ /** The action for which we are storing the stat. */
+ public Action getAction() {
+ return action;
+ }
+
+ /**
+ * Add statistics for one dependency of this action.
+ */
+ public void addDepInfo(C dep) {
+ Preconditions.checkState(!dep.isRunning,
+ "Cannot add critical path stats when the action is not finished. %s. %s", action,
+ dep.getAction());
+ long childAggregatedWallTime = dep.getAggregatedWallTime();
+ // Replace the child if its critical path had the maximum wall time.
+ if (child == null || childAggregatedWallTime > this.childAggregatedWallTime) {
+ this.childAggregatedWallTime = childAggregatedWallTime;
+ child = dep;
+ }
+ }
+
+ public long getActionWallTime() {
+ Preconditions.checkState(!isRunning, "Still running %s", action);
+ return finishTime - startTime;
+ }
+
+ /**
+ * Returns the current critical path for the action in milliseconds.
+ *
+ * <p>Critical path is defined as : action_execution_time + max(child_critical_path).
+ */
+ public long getAggregatedWallTime() {
+ Preconditions.checkState(!isRunning, "Still running %s", action);
+ return getActionWallTime() + childAggregatedWallTime;
+ }
+
+ /** Time when the action started to execute. Milliseconds since epoch time. */
+ public long getStartTime() {
+ return startTime;
+ }
+
+ /**
+ * Get the child critical path component.
+ *
+ * <p>The component dependency with the maximum total critical path time.
+ */
+ @Nullable
+ public C getChild() {
+ return child;
+ }
+
+ /**
+ * Returns a human readable representation of the critical path stats with all the details.
+ */
+ @Override
+ public String toString() {
+ String currentTime = "still running ";
+ if (!isRunning) {
+ currentTime = String.format("%.2f", getActionWallTime() / 1000.0) + "s ";
+ }
+ return currentTime + action.describe();
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java b/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java
new file mode 100644
index 0000000..dd70c35
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java
@@ -0,0 +1,70 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+
+/**
+ * Aggregates all the critical path components in one object. This allows us to easily access the
+ * components data and have a proper toString().
+ */
+public class AggregatedCriticalPath<T extends AbstractCriticalPathComponent> {
+
+ private final long totalTime;
+ private final ImmutableList<T> criticalPathComponents;
+
+ protected AggregatedCriticalPath(long totalTime, ImmutableList<T> criticalPathComponents) {
+ this.totalTime = totalTime;
+ this.criticalPathComponents = criticalPathComponents;
+ }
+
+ /** Total wall time in ms spent running the critical path actions. */
+ public long totalTime() {
+ return totalTime;
+ }
+
+ /** Returns a list of all the component stats for the critical path. */
+ public ImmutableList<T> components() {
+ return criticalPathComponents;
+ }
+
+ @Override
+ public String toString() {
+ return toString(false);
+ }
+
+ /**
+ * Returns a summary version of the critical path stats that omits stats that are not useful
+ * to the user.
+ */
+ public String toStringSummary() {
+ return toString(true);
+ }
+
+ private String toString(boolean summary) {
+ StringBuilder sb = new StringBuilder("Critical Path: ");
+ double totalMillis = totalTime;
+ sb.append(String.format("%.2f", totalMillis / 1000.0));
+ sb.append("s");
+ if (summary || criticalPathComponents.isEmpty()) {
+ return sb.toString();
+ }
+ sb.append("\n ");
+ Joiner.on("\n ").appendTo(sb, criticalPathComponents);
+ return sb.toString();
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java b/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java
new file mode 100644
index 0000000..cc240c4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java
@@ -0,0 +1,255 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.AllowConcurrentEvents;
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.Subscribe;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.AnalysisFailureEvent;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
+import com.google.devtools.build.lib.analysis.TargetCompleteEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.TestFilteringCompleteEvent;
+import com.google.devtools.build.lib.concurrent.ThreadSafety;
+import com.google.devtools.build.lib.events.ExceptionListener;
+import com.google.devtools.build.lib.rules.test.TestProvider;
+import com.google.devtools.build.lib.rules.test.TestResult;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * This class aggregates and reports target-wide test statuses in real-time.
+ * It must be public for EventBus invocation.
+ */
+@ThreadSafety.ThreadSafe
+public class AggregatingTestListener {
+ private final ConcurrentMap<Artifact, TestResult> statusMap = new MapMaker().makeMap();
+
+ private final TestResultAnalyzer analyzer;
+ private final EventBus eventBus;
+ private final EventHandlerPreconditions preconditionHelper;
+ private volatile boolean blazeHalted = false;
+
+
+ // summaryLock guards concurrent access to these two collections, which should be kept
+ // synchronized with each other.
+ private final Map<LabelAndConfiguration, TestSummary.Builder> summaries;
+ private final Multimap<LabelAndConfiguration, Artifact> remainingRuns;
+ private final Object summaryLock = new Object();
+
+ public AggregatingTestListener(TestResultAnalyzer analyzer,
+ EventBus eventBus,
+ ExceptionListener listener) {
+ this.analyzer = analyzer;
+ this.eventBus = eventBus;
+ this.preconditionHelper = new EventHandlerPreconditions(listener);
+
+ this.summaries = Maps.newHashMap();
+ this.remainingRuns = HashMultimap.create();
+ }
+
+ /**
+ * @return An unmodifiable copy of the map of test results.
+ */
+ public Map<Artifact, TestResult> getStatusMap() {
+ return ImmutableMap.copyOf(statusMap);
+ }
+
+ /**
+ * Populates the test summary map as soon as test filtering is complete.
+ * This is the earliest at which the final set of targets to test is known.
+ */
+ @Subscribe
+ @AllowConcurrentEvents
+ public void populateTests(TestFilteringCompleteEvent event) {
+ // Add all target runs to the map, assuming 1:1 status artifact <-> result.
+ synchronized (summaryLock) {
+ for (ConfiguredTarget target : event.getTestTargets()) {
+ Iterable<Artifact> statusArtifacts =
+ target.getProvider(TestProvider.class).getTestParams().getTestStatusArtifacts();
+ preconditionHelper.checkState(remainingRuns.putAll(asKey(target), statusArtifacts));
+
+ // And create an empty summary suitable for incremental analysis.
+ // Also has the nice side effect of mapping labels to RuleConfiguredTargets.
+ TestSummary.Builder summary = TestSummary.newBuilder()
+ .setTarget(target)
+ .setStatus(BlazeTestStatus.NO_STATUS);
+ preconditionHelper.checkState(summaries.put(asKey(target), summary) == null);
+ }
+ }
+ }
+
+ /**
+ * Records a new test run result and incrementally updates the target status.
+ * This event is sent upon completion of executed test runs.
+ */
+ @Subscribe
+ @AllowConcurrentEvents
+ public void testEvent(TestResult result) {
+ Preconditions.checkState(
+ statusMap.put(result.getTestStatusArtifact(), result) == null,
+ "Duplicate result reported for an individual test shard");
+
+ ActionOwner testOwner = result.getTestAction().getOwner();
+ LabelAndConfiguration targetLabel = LabelAndConfiguration.of(
+ testOwner.getLabel(), result.getTestAction().getConfiguration());
+
+ TestSummary finalTestSummary = null;
+ synchronized (summaryLock) {
+ TestSummary.Builder summary = summaries.get(targetLabel);
+ preconditionHelper.checkNotNull(summary);
+ if (!remainingRuns.remove(targetLabel, result.getTestStatusArtifact())) {
+ // This can happen if a buildCompleteEvent() was processed before this event reached us.
+ // This situation is likely to happen if --notest_keep_going is set with multiple targets.
+ return;
+ }
+
+ summary = analyzer.incrementalAnalyze(summary, result);
+
+ // If all runs are processed, the target is finished and ready to report.
+ if (!remainingRuns.containsKey(targetLabel)) {
+ finalTestSummary = summary.build();
+ }
+ }
+
+ // Report finished targets.
+ if (finalTestSummary != null) {
+ eventBus.post(finalTestSummary);
+ }
+ }
+
+ private void targetFailure(LabelAndConfiguration label) {
+ TestSummary finalSummary;
+ synchronized (summaryLock) {
+ if (!remainingRuns.containsKey(label)) {
+ // Blaze does not guarantee that BuildResult.getSuccessfulTargets() and posted TestResult
+ // events are in sync. Thus, it is possible that a test event was posted, but the target is
+ // not present in the set of successful targets.
+ return;
+ }
+
+ TestSummary.Builder summary = summaries.get(label);
+ if (summary == null) {
+ // Not a test target; nothing to do.
+ return;
+ }
+ finalSummary = analyzer.markUnbuilt(summary, blazeHalted).build();
+
+ // These are never going to run; removing them marks the target complete.
+ remainingRuns.removeAll(label);
+ }
+ eventBus.post(finalSummary);
+ }
+
+ @VisibleForTesting
+ void buildComplete(
+ Collection<ConfiguredTarget> actualTargets, Collection<ConfiguredTarget> successfulTargets) {
+ if (actualTargets == null || successfulTargets == null) {
+ return;
+ }
+
+ for (ConfiguredTarget target: Sets.difference(
+ ImmutableSet.copyOf(actualTargets), ImmutableSet.copyOf(successfulTargets))) {
+ targetFailure(asKey(target));
+ }
+ }
+
+ @Subscribe
+ public void buildCompleteEvent(BuildCompleteEvent event) {
+ if (event.getResult().wasCatastrophe()) {
+ blazeHalted = true;
+ }
+ buildComplete(event.getResult().getActualTargets(), event.getResult().getSuccessfulTargets());
+ }
+
+ @Subscribe
+ public void analysisFailure(AnalysisFailureEvent event) {
+ targetFailure(event.getFailedTarget());
+ }
+
+ @Subscribe
+ @AllowConcurrentEvents
+ public void buildInterrupted(BuildInterruptedEvent event) {
+ blazeHalted = true;
+ }
+
+ /**
+ * Called when a build action is not executed (e.g. because a dependency failed to build). We want
+ * to catch such events in order to determine when a test target has failed to build.
+ */
+ @Subscribe
+ @AllowConcurrentEvents
+ public void targetComplete(TargetCompleteEvent event) {
+ if (event.failed()) {
+ targetFailure(new LabelAndConfiguration(event.getTarget()));
+ }
+ }
+
+ /**
+ * Returns the known aggregate results for the given target at the current moment.
+ */
+ public TestSummary.Builder getCurrentSummary(ConfiguredTarget target) {
+ synchronized (summaryLock) {
+ return summaries.get(asKey(target));
+ }
+ }
+
+ /**
+ * Returns all test status artifacts associated with a given target
+ * whose runs have yet to finish.
+ */
+ public Collection<Artifact> getIncompleteRuns(ConfiguredTarget target) {
+ synchronized (summaryLock) {
+ return Collections.unmodifiableCollection(remainingRuns.get(asKey(target)));
+ }
+ }
+
+ /**
+ * Returns true iff all runs of the target are accounted for.
+ */
+ public boolean targetReported(ConfiguredTarget target) {
+ synchronized (summaryLock) {
+ return summaries.containsKey(asKey(target)) && !remainingRuns.containsKey(asKey(target));
+ }
+ }
+
+ /**
+ * Returns the {@link TestResultAnalyzer} associated with this listener.
+ */
+ public TestResultAnalyzer getAnalyzer() {
+ return analyzer;
+ }
+
+ private LabelAndConfiguration asKey(ConfiguredTarget target) {
+ return new LabelAndConfiguration(target);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java
new file mode 100644
index 0000000..61f46a8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java
@@ -0,0 +1,63 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+/**
+ * Interface implemented by Blaze commands. In addition to implementing this interface, each
+ * command must be annotated with a {@link Command} annotation.
+ */
+public interface BlazeCommand {
+ /**
+ * This method provides the imperative portion of the command. It takes
+ * a {@link OptionsProvider} instance {@code options}, which provides access
+ * to the options instances via {@link OptionsProvider#getOptions(Class)},
+ * and access to the residue (the remainder of the command line) via
+ * {@link OptionsProvider#getResidue()}. The framework parses and makes
+ * available exactly the options that the command class specifies via the
+ * annotation {@link Command#options()}. The command may write to standard
+ * out and standard error via {@code outErr}. It indicates success / failure
+ * via its return value, which becomes the Unix exit status of the Blaze
+ * client process. It may indicate a shutdown request by throwing
+ * {@link BlazeCommandDispatcher.ShutdownBlazeServerException}. In that case,
+ * the Blaze server process (the memory resident portion of Blaze) will
+ * shut down and the exit status will be 0 (in case the shutdown succeeds
+ * without error).
+ *
+ * @param runtime The Blaze runtime requesting the execution of the command
+ * @param options A parsed options instance initialized with the values for
+ * the options specified in {@link Command#options()}.
+ *
+ * @return The Unix exit status for the Blaze client.
+ * @throws BlazeCommandDispatcher.ShutdownBlazeServerException Indicates
+ * that the command wants to shutdown the Blaze server.
+ */
+ ExitCode exec(BlazeRuntime runtime, OptionsProvider options)
+ throws BlazeCommandDispatcher.ShutdownBlazeServerException;
+
+ /**
+ * Allows the command to provide command-specific option defaults and/or
+ * requirements. This method is called after all command-line and rc file options have been
+ * parsed.
+ *
+ * @param runtime The Blaze runtime requesting the execution of the command
+ *
+ * @throws AbruptExitException if something went wrong
+ */
+ void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) throws AbruptExitException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java
new file mode 100644
index 0000000..cee47ee
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java
@@ -0,0 +1,692 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.io.Flushables;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.AnsiStrippingOutputStream;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.DelegatingOutErr;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.common.options.OptionPriority;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.logging.Level;
+
+/**
+ * Dispatches to the Blaze commands; that is, given a command line, this
+ * abstraction looks up the appropriate command object, parses the options
+ * required by the object, and calls its exec method. Also, this object provides
+ * the runtime state (BlazeRuntime) to the commands.
+ */
+public class BlazeCommandDispatcher {
+
+ // Keep in sync with options added in OptionProcessor::AddRcfileArgsAndOptions()
+ private static final Set<String> INTERNAL_COMMAND_OPTIONS = ImmutableSet.of(
+ "rc_source", "default_override", "isatty", "terminal_columns", "ignore_client_env",
+ "client_env", "client_cwd");
+
+ private static final ImmutableList<String> HELP_COMMAND = ImmutableList.of("help");
+
+ private static final Set<String> ALL_HELP_OPTIONS = ImmutableSet.of("--help", "-help", "-h");
+
+ /**
+ * By throwing this exception, a command indicates that it wants to shutdown
+ * the Blaze server process.
+ * See {@link BlazeCommandDispatcher#exec(List, OutErr, long)}.
+ */
+ public static class ShutdownBlazeServerException extends Exception {
+ private final int exitStatus;
+
+ public ShutdownBlazeServerException(int exitStatus, Throwable cause) {
+ super(cause);
+ this.exitStatus = exitStatus;
+ }
+
+ public ShutdownBlazeServerException(int exitStatus) {
+ this.exitStatus = exitStatus;
+ }
+
+ public int getExitStatus() {
+ return exitStatus;
+ }
+ }
+
+ private final BlazeRuntime runtime;
+ private final Map<String, BlazeCommand> commandsByName = new LinkedHashMap<>();
+
+ private OutputStream logOutputStream = null;
+
+ /**
+ * Create a Blaze dispatcher that uses the specified {@code BlazeRuntime}
+ * instance, and no default options, and delegates to {@code commands} as
+ * appropriate.
+ */
+ @VisibleForTesting
+ public BlazeCommandDispatcher(BlazeRuntime runtime, BlazeCommand... commands) {
+ this(runtime, ImmutableList.copyOf(commands));
+ }
+
+ /**
+ * Create a Blaze dispatcher that uses the specified {@code BlazeRuntime}
+ * instance, and delegates to {@code commands} as appropriate.
+ */
+ public BlazeCommandDispatcher(BlazeRuntime runtime, Iterable<BlazeCommand> commands) {
+ this.runtime = runtime;
+ for (BlazeCommand command : commands) {
+ addCommandByName(command);
+ }
+
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ for (BlazeCommand command : module.getCommands()) {
+ addCommandByName(command);
+ }
+ }
+
+ runtime.setCommandMap(commandsByName);
+ }
+
+ /**
+ * Adds the given command under the given name to the map of commands.
+ *
+ * @throws AssertionError if the name is already used by another command.
+ */
+ private void addCommandByName(BlazeCommand command) {
+ String name = command.getClass().getAnnotation(Command.class).name();
+ if (commandsByName.containsKey(name)) {
+ throw new IllegalStateException("Command name or alias " + name + " is already used.");
+ }
+ commandsByName.put(name, command);
+ }
+
+ /**
+ * Only some commands work if cwd != workspaceSuffix in Blaze. In that case, also check if Blaze
+ * was called from the output directory and fail if it was.
+ */
+ private ExitCode checkCwdInWorkspace(Command commandAnnotation, String commandName,
+ OutErr outErr) {
+ if (!commandAnnotation.mustRunInWorkspace()) {
+ return ExitCode.SUCCESS;
+ }
+
+ if (!runtime.inWorkspace()) {
+ outErr.printErrLn("The '" + commandName + "' command is only supported from within a "
+ + "workspace.");
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ Path workspace = runtime.getWorkspace();
+ Path doNotBuild = workspace.getParentDirectory().getRelative(
+ BlazeRuntime.DO_NOT_BUILD_FILE_NAME);
+ if (doNotBuild.exists()) {
+ if (!commandAnnotation.canRunInOutputDirectory()) {
+ outErr.printErrLn(getNotInRealWorkspaceError(doNotBuild));
+ return ExitCode.COMMAND_LINE_ERROR;
+ } else {
+ outErr.printErrLn("WARNING: Blaze is run from output directory. This is unsound.");
+ }
+ }
+ return ExitCode.SUCCESS;
+ }
+
+ private CommonCommandOptions checkOptions(OptionsParser optionsParser,
+ Command commandAnnotation, List<String> args, List<String> rcfileNotes, OutErr outErr)
+ throws OptionsParsingException {
+ Function<String, String> commandOptionSourceFunction = new Function<String, String>() {
+ @Override
+ public String apply(String input) {
+ if (INTERNAL_COMMAND_OPTIONS.contains(input)) {
+ return "options generated by Blaze launcher";
+ } else {
+ return "command line options";
+ }
+ }
+ };
+
+ // Explicit command-line options:
+ List<String> cmdLineAfterCommand = args.subList(1, args.size());
+ optionsParser.parseWithSourceFunction(OptionPriority.COMMAND_LINE,
+ commandOptionSourceFunction, cmdLineAfterCommand);
+
+ // Command-specific options from .blazerc passed in via --default_override
+ // and --rc_source. A no-op if none are provided.
+ CommonCommandOptions rcFileOptions = optionsParser.getOptions(CommonCommandOptions.class);
+ List<Pair<String, ListMultimap<String, String>>> optionsMap =
+ getOptionsMap(outErr, rcFileOptions.rcSource, rcFileOptions.optionsOverrides,
+ commandsByName.keySet());
+
+ parseOptionsForCommand(rcfileNotes, commandAnnotation, optionsParser, optionsMap, null);
+
+ // Fix-point iteration until all configs are loaded.
+ List<String> configsLoaded = ImmutableList.of();
+ CommonCommandOptions commonOptions = optionsParser.getOptions(CommonCommandOptions.class);
+ while (!commonOptions.configs.equals(configsLoaded)) {
+ Set<String> missingConfigs = new LinkedHashSet<>(commonOptions.configs);
+ missingConfigs.removeAll(configsLoaded);
+ parseOptionsForCommand(rcfileNotes, commandAnnotation, optionsParser, optionsMap,
+ missingConfigs);
+ configsLoaded = commonOptions.configs;
+ commonOptions = optionsParser.getOptions(CommonCommandOptions.class);
+ }
+
+ return commonOptions;
+ }
+
+ /**
+ * Sends {@code EventKind.{STDOUT|STDERR}} messages to the given {@link OutErr}.
+ *
+ * <p>This is necessary because we cannot delete the output files from the previous Blaze run
+ * because there can be processes spawned by the previous invocation that are still processing
+ * them, in which case we need to print a warning message about that.
+ *
+ * <p>Thus, messages sent to {@link Reporter#getOutErr} get sent to this event handler, then
+ * to its {@link OutErr}. We need to go deeper!
+ */
+ private static class OutErrEventHandler implements EventHandler {
+ private final OutErr outErr;
+
+ private OutErrEventHandler(OutErr outErr) {
+ this.outErr = outErr;
+ }
+
+ @Override
+ public void handle(Event event) {
+ try {
+ switch (event.getKind()) {
+ case STDOUT:
+ outErr.getOutputStream().write(event.getMessageBytes());
+ break;
+ case STDERR:
+ outErr.getErrorStream().write(event.getMessageBytes());
+ break;
+ }
+ } catch (IOException e) {
+ // We cannot do too much here -- ErrorEventListener#handle does not provide us with ways to
+ // report an error.
+ }
+ }
+ }
+
+ /**
+ * Executes a single command. Returns the Unix exit status for the Blaze
+ * client process, or throws {@link ShutdownBlazeServerException} to
+ * indicate that a command wants to shutdown the Blaze server.
+ */
+ public int exec(List<String> args, OutErr originalOutErr, long firstContactTime)
+ throws ShutdownBlazeServerException {
+ // Record the start time for the profiler and the timestamp granularity monitor. Do not put
+ // anything before this!
+ long execStartTimeNanos = runtime.getClock().nanoTime();
+
+ // Record the command's starting time for use by the commands themselves.
+ runtime.recordCommandStartTime(firstContactTime);
+
+ // Record the command's starting time again, for use by
+ // TimestampGranularityMonitor.waitForTimestampGranularity().
+ // This should be done as close as possible to the start of
+ // the command's execution - that's why we do this separately,
+ // rather than in runtime.beforeCommand().
+ runtime.getTimestampGranularityMonitor().setCommandStartTime();
+ runtime.initEventBus();
+
+ // Give a chance for module.beforeCommand() to report an errors to stdout and stderr.
+ // Once we can close the old streams, this event handler is removed.
+ OutErrEventHandler originalOutErrEventHandler =
+ new OutErrEventHandler(originalOutErr);
+ runtime.getReporter().addHandler(originalOutErrEventHandler);
+ OutErr outErr = originalOutErr;
+ runtime.getReporter().removeHandler(originalOutErrEventHandler);
+
+ if (args.isEmpty()) { // Default to help command if no arguments specified.
+ args = HELP_COMMAND;
+ }
+ String commandName = args.get(0);
+
+ // Be gentle to users who want to find out about Blaze invocation.
+ if (ALL_HELP_OPTIONS.contains(commandName)) {
+ commandName = "help";
+ }
+
+ BlazeCommand command = commandsByName.get(commandName);
+ if (command == null) {
+ outErr.printErrLn("Command '" + commandName + "' not found. " + "Try 'blaze help'.");
+ return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode();
+ }
+ Command commandAnnotation = command.getClass().getAnnotation(Command.class);
+
+ AbruptExitException exitCausingException = null;
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ try {
+ module.beforeCommand(runtime, commandAnnotation);
+ } catch (AbruptExitException e) {
+ // Don't let one module's complaints prevent the other modules from doing necessary
+ // setup. We promised to call beforeCommand exactly once per-module before each command
+ // and will be calling afterCommand soon in the future - a module's afterCommand might
+ // rightfully assume its beforeCommand has already been called.
+ outErr.printErrLn(e.getMessage());
+ // It's not ideal but we can only return one exit code, so we just pick the code of the
+ // last exception.
+ exitCausingException = e;
+ }
+ }
+ if (exitCausingException != null) {
+ return exitCausingException.getExitCode().getNumericExitCode();
+ }
+
+ try {
+ Path commandLog = getCommandLogPath(runtime.getOutputBase());
+
+ // Unlink old command log from previous build, if present, so scripts
+ // reading it don't conflate it with the command log we're about to write.
+ commandLog.delete();
+
+ logOutputStream = commandLog.getOutputStream();
+ outErr = tee(originalOutErr, OutErr.create(logOutputStream, logOutputStream));
+ } catch (IOException ioException) {
+ LoggingUtil.logToRemote(
+ Level.WARNING, "Unable to delete or open command.log", ioException);
+ }
+
+ // Create the UUID for this command.
+ runtime.setCommandId(UUID.randomUUID());
+
+ ExitCode result = checkCwdInWorkspace(commandAnnotation, commandName, outErr);
+ if (result != ExitCode.SUCCESS) {
+ return result.getNumericExitCode();
+ }
+
+ OptionsParser optionsParser;
+ CommonCommandOptions commonOptions;
+ // Delay output of notes regarding the parsed rc file, so it's possible to disable this in the
+ // rc file.
+ List<String> rcfileNotes = new ArrayList<>();
+ try {
+ optionsParser = createOptionsParser(command);
+ commonOptions = checkOptions(optionsParser, commandAnnotation, args, rcfileNotes, outErr);
+ } catch (OptionsParsingException e) {
+ for (String note : rcfileNotes) {
+ outErr.printErrLn("INFO: " + note);
+ }
+ outErr.printErrLn(e.getMessage());
+ return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode();
+ }
+
+ // Setup log filtering
+ BlazeCommandEventHandler.Options eventHandlerOptions =
+ optionsParser.getOptions(BlazeCommandEventHandler.Options.class);
+ if (!eventHandlerOptions.useColor()) {
+ if (!commandAnnotation.binaryStdOut()) {
+ outErr = ansiStripOut(outErr);
+ }
+
+ if (!commandAnnotation.binaryStdErr()) {
+ outErr = ansiStripErr(outErr);
+ }
+ }
+
+ BlazeRuntime.setupLogging(commonOptions.verbosity);
+
+ // Do this before an actual crash so we don't have to worry about
+ // allocating memory post-crash.
+ String[] crashData = runtime.getCrashData();
+ int numericExitCode = ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode();
+ PrintStream savedOut = System.out;
+ PrintStream savedErr = System.err;
+
+ EventHandler handler = createEventHandler(outErr, eventHandlerOptions);
+ Reporter reporter = runtime.getReporter();
+ reporter.addHandler(handler);
+ try {
+ // While a Blaze command is active, direct all errors to the client's
+ // event handler (and out/err streams).
+ OutErr reporterOutErr = reporter.getOutErr();
+ System.setOut(new PrintStream(reporterOutErr.getOutputStream(), /*autoflush=*/true));
+ System.setErr(new PrintStream(reporterOutErr.getErrorStream(), /*autoflush=*/true));
+
+ if (commonOptions.announceRcOptions) {
+ for (String note : rcfileNotes) {
+ reporter.handle(Event.info(note));
+ }
+ }
+
+ try {
+ // Notify the BlazeRuntime, so it can do some initial setup.
+ runtime.beforeCommand(commandName, optionsParser, commonOptions, execStartTimeNanos);
+ // Allow the command to edit options after parsing:
+ command.editOptions(runtime, optionsParser);
+ } catch (AbruptExitException e) {
+ reporter.handle(Event.error(e.getMessage()));
+ return e.getExitCode().getNumericExitCode();
+ }
+
+ // Print warnings for odd options usage
+ for (String warning : optionsParser.getWarnings()) {
+ reporter.handle(Event.warn(warning));
+ }
+
+ ExitCode outcome = command.exec(runtime, optionsParser);
+ outcome = runtime.precompleteCommand(outcome);
+ numericExitCode = outcome.getNumericExitCode();
+ return numericExitCode;
+ } catch (ShutdownBlazeServerException e) {
+ numericExitCode = e.getExitStatus();
+ throw e;
+ } catch (Throwable e) {
+ BugReport.printBug(outErr, e);
+ BugReport.sendBugReport(e, args, crashData);
+ numericExitCode = e instanceof OutOfMemoryError
+ ? ExitCode.OOM_ERROR.getNumericExitCode()
+ : ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode();
+ throw new ShutdownBlazeServerException(numericExitCode, e);
+ } finally {
+ runtime.afterCommand(numericExitCode);
+ // Swallow IOException, as we are already in a finally clause
+ Flushables.flushQuietly(outErr.getOutputStream());
+ Flushables.flushQuietly(outErr.getErrorStream());
+
+ System.setOut(savedOut);
+ System.setErr(savedErr);
+ reporter.removeHandler(handler);
+ releaseHandler(handler);
+ runtime.getTimestampGranularityMonitor().waitForTimestampGranularity(outErr);
+ }
+ }
+
+ /**
+ * For testing ONLY. Same as {@link #exec(List, OutErr, long)}, but automatically uses the current
+ * time.
+ */
+ @VisibleForTesting
+ public int exec(List<String> args, OutErr originalOutErr) throws ShutdownBlazeServerException {
+ return exec(args, originalOutErr, runtime.getClock().currentTimeMillis());
+ }
+
+ /**
+ * Parses the options from .rc files for a command invocation. It works in one of two modes;
+ * either it loads the non-config options, or the config options that are specified in the {@code
+ * configs} parameter.
+ *
+ * <p>This method adds every option pertaining to the specified command to the options parser. To
+ * do that, it needs the command -> option mapping that is generated from the .rc files.
+ *
+ * <p>It is not as trivial as simply taking the list of options for the specified command because
+ * commands can inherit arguments from each other, and we have to respect that (e.g. if an option
+ * is specified for 'build', it needs to take effect for the 'test' command, too).
+ *
+ * <p>Note that the order in which the options are parsed is well-defined: all options from the
+ * same rc file are parsed at the same time, and the rc files are handled in the order in which
+ * they were passed in from the client.
+ *
+ * @param rcfileNotes note message that would be printed during parsing
+ * @param commandAnnotation the command for which options should be parsed.
+ * @param optionsParser parser to receive parsed options.
+ * @param optionsMap .rc files in structured format: a list of pairs, where the first part is the
+ * name of the rc file, and the second part is a multimap of command name (plus config, if
+ * present) to the list of options for that command
+ * @param configs the configs for which to parse options; if {@code null}, non-config options are
+ * parsed
+ * @throws OptionsParsingException
+ */
+ protected static void parseOptionsForCommand(List<String> rcfileNotes, Command commandAnnotation,
+ OptionsParser optionsParser, List<Pair<String, ListMultimap<String, String>>> optionsMap,
+ Iterable<String> configs) throws OptionsParsingException {
+ for (String commandToParse : getCommandNamesToParse(commandAnnotation)) {
+ for (Pair<String, ListMultimap<String, String>> entry : optionsMap) {
+ List<String> allOptions = new ArrayList<>();
+ if (configs == null) {
+ allOptions.addAll(entry.second.get(commandToParse));
+ } else {
+ for (String config : configs) {
+ allOptions.addAll(entry.second.get(commandToParse + ":" + config));
+ }
+ }
+ processOptionList(optionsParser, commandToParse,
+ commandAnnotation.name(), rcfileNotes, entry.first, allOptions);
+ if (allOptions.isEmpty()) {
+ continue;
+ }
+ }
+ }
+ }
+
+ // Processes the option list for an .rc file - command pair.
+ private static void processOptionList(OptionsParser optionsParser, String commandToParse,
+ String originalCommand, List<String> rcfileNotes, String rcfile, List<String> rcfileOptions)
+ throws OptionsParsingException {
+ if (!rcfileOptions.isEmpty()) {
+ String inherited = commandToParse.equals(originalCommand) ? "" : "Inherited ";
+ rcfileNotes.add("Reading options for '" + originalCommand +
+ "' from " + rcfile + ":\n" +
+ " " + inherited + "'" + commandToParse + "' options: "
+ + Joiner.on(' ').join(rcfileOptions));
+ optionsParser.parse(OptionPriority.RC_FILE, rcfile, rcfileOptions);
+ }
+ }
+
+ private static List<String> getCommandNamesToParse(Command commandAnnotation) {
+ List<String> result = new ArrayList<>();
+ getCommandNamesToParseHelper(commandAnnotation, result);
+ result.add("common");
+ // TODO(bazel-team): This statement is a NO-OP: Lists.reverse(result);
+ return result;
+ }
+
+ private static void getCommandNamesToParseHelper(Command commandAnnotation,
+ List<String> accumulator) {
+ for (Class<? extends BlazeCommand> base : commandAnnotation.inherits()) {
+ getCommandNamesToParseHelper(base.getAnnotation(Command.class), accumulator);
+ }
+ accumulator.add(commandAnnotation.name());
+ }
+
+ private OutErr ansiStripOut(OutErr outErr) {
+ OutputStream wrappedOut = new AnsiStrippingOutputStream(outErr.getOutputStream());
+ return OutErr.create(wrappedOut, outErr.getErrorStream());
+ }
+
+ private OutErr ansiStripErr(OutErr outErr) {
+ OutputStream wrappedErr = new AnsiStrippingOutputStream(outErr.getErrorStream());
+ return OutErr.create(outErr.getOutputStream(), wrappedErr);
+ }
+
+ private String getNotInRealWorkspaceError(Path doNotBuildFile) {
+ String message = "Blaze should not be called from a Blaze output directory. ";
+ try {
+ String realWorkspace =
+ new String(FileSystemUtils.readContentAsLatin1(doNotBuildFile));
+ message += String.format("The pertinent workspace directory is: '%s'",
+ realWorkspace);
+ } catch (IOException e) {
+ // We are exiting anyway.
+ }
+
+ return message;
+ }
+
+ /**
+ * For a given output_base directory, returns the command log file path.
+ */
+ public static Path getCommandLogPath(Path outputBase) {
+ return outputBase.getRelative("command.log");
+ }
+
+ private OutErr tee(OutErr outErr1, OutErr outErr2) {
+ DelegatingOutErr outErr = new DelegatingOutErr();
+ outErr.addSink(outErr1);
+ outErr.addSink(outErr2);
+ return outErr;
+ }
+
+ private void closeSilently(OutputStream logOutputStream) {
+ if (logOutputStream != null) {
+ try {
+ logOutputStream.close();
+ } catch (IOException e) {
+ LoggingUtil.logToRemote(Level.WARNING, "Unable to close command.log", e);
+ }
+ }
+ }
+
+ /**
+ * Creates an option parser using the common options classes and the
+ * command-specific options classes.
+ *
+ * <p>An overriding method should first call this method and can then
+ * override default values directly or by calling {@link
+ * #parseOptionsForCommand} for command-specific options.
+ *
+ * @throws OptionsParsingException
+ */
+ protected OptionsParser createOptionsParser(BlazeCommand command)
+ throws OptionsParsingException {
+ Command annotation = command.getClass().getAnnotation(Command.class);
+ List<Class<? extends OptionsBase>> allOptions = Lists.newArrayList();
+ allOptions.addAll(BlazeCommandUtils.getOptions(
+ command.getClass(), getRuntime().getBlazeModules(), getRuntime().getRuleClassProvider()));
+ OptionsParser parser = OptionsParser.newOptionsParser(allOptions);
+ parser.setAllowResidue(annotation.allowResidue());
+ return parser;
+ }
+
+ /**
+ * Convert a list of option override specifications to a more easily digestible
+ * form.
+ *
+ * @param overrides list of option override specifications
+ */
+ @VisibleForTesting
+ static List<Pair<String, ListMultimap<String, String>>> getOptionsMap(
+ OutErr outErr,
+ List<String> rcFiles,
+ List<CommonCommandOptions.OptionOverride> overrides,
+ Set<String> validCommands) {
+ List<Pair<String, ListMultimap<String, String>>> result = new ArrayList<>();
+
+ String lastRcFile = null;
+ ListMultimap<String, String> lastMap = null;
+ for (CommonCommandOptions.OptionOverride override : overrides) {
+ if (override.blazeRc < 0 || override.blazeRc >= rcFiles.size()) {
+ outErr.printErrLn("WARNING: inconsistency in generated command line "
+ + "args. Ignoring bogus argument\n");
+ continue;
+ }
+ String rcFile = rcFiles.get(override.blazeRc);
+
+ String command = override.command;
+ int index = command.indexOf(':');
+ if (index > 0) {
+ command = command.substring(0, index);
+ }
+ if (!validCommands.contains(command) && !command.equals("common")) {
+ outErr.printErrLn("WARNING: while reading option defaults file '"
+ + rcFile + "':\n"
+ + " invalid command name '" + override.command + "'.");
+ continue;
+ }
+
+ if (!rcFile.equals(lastRcFile)) {
+ if (lastRcFile != null) {
+ result.add(Pair.of(lastRcFile, lastMap));
+ }
+ lastRcFile = rcFile;
+ lastMap = ArrayListMultimap.create();
+ }
+ lastMap.put(override.command, override.option);
+ }
+ if (lastRcFile != null) {
+ result.add(Pair.of(lastRcFile, lastMap));
+ }
+
+ return result;
+ }
+
+ /**
+ * Returns the event handler to use for this Blaze command.
+ */
+ private EventHandler createEventHandler(OutErr outErr,
+ BlazeCommandEventHandler.Options eventOptions) {
+ EventHandler eventHandler;
+ if ((eventOptions.useColor() || eventOptions.useCursorControl())) {
+ eventHandler = new FancyTerminalEventHandler(outErr, eventOptions);
+ } else {
+ eventHandler = new BlazeCommandEventHandler(outErr, eventOptions);
+ }
+
+ return RateLimitingEventHandler.create(eventHandler, eventOptions.showProgressRateLimit);
+ }
+
+ /**
+ * Unsets the event handler.
+ */
+ private void releaseHandler(EventHandler eventHandler) {
+ if (eventHandler instanceof FancyTerminalEventHandler) {
+ // Make sure that the terminal state of the old event handler is clear
+ // before creating a new one.
+ ((FancyTerminalEventHandler)eventHandler).resetTerminal();
+ }
+ }
+
+ /**
+ * Returns the runtime instance shared by the commands that this dispatcher
+ * dispatches to.
+ */
+ public BlazeRuntime getRuntime() {
+ return runtime;
+ }
+
+ /**
+ * The map from command names to commands that this dispatcher dispatches to.
+ */
+ Map<String, BlazeCommand> getCommandsByName() {
+ return Collections.unmodifiableMap(commandsByName);
+ }
+
+ /**
+ * Shuts down all the registered commands to give them a chance to cleanup or
+ * close resources. Should be called by the owner of this command dispatcher
+ * in all termination cases.
+ */
+ public void shutdown() {
+ closeSilently(logOutputStream);
+ logOutputStream = null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java
new file mode 100644
index 0000000..603b0be
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java
@@ -0,0 +1,246 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Location;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.common.options.EnumConverter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.EnumSet;
+import java.util.Set;
+
+/**
+ * BlazeCommandEventHandler: an event handler established for the duration of a
+ * single Blaze command.
+ */
+public class BlazeCommandEventHandler implements EventHandler {
+
+ public enum UseColor { YES, NO, AUTO }
+ public enum UseCurses { YES, NO, AUTO }
+
+ public static class UseColorConverter extends EnumConverter<UseColor> {
+ public UseColorConverter() {
+ super(UseColor.class, "--color setting");
+ }
+ }
+
+ public static class UseCursesConverter extends EnumConverter<UseCurses> {
+ public UseCursesConverter() {
+ super(UseCurses.class, "--curses setting");
+ }
+ }
+
+ public static class Options extends OptionsBase {
+
+ @Option(name = "show_progress",
+ defaultValue = "true",
+ category = "verbosity",
+ help = "Display progress messages during a build.")
+ public boolean showProgress;
+
+ @Option(name = "show_task_finish",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Display progress messages when tasks complete, not just when they start.")
+ public boolean showTaskFinish;
+
+ @Option(name = "show_progress_rate_limit",
+ defaultValue = "0.03", // A nice middle ground; snappy but not too spammy in logs.
+ category = "verbosity",
+ help = "Minimum number of seconds between progress messages in the output.")
+ public double showProgressRateLimit;
+
+ @Option(name = "color",
+ defaultValue = "auto",
+ converter = UseColorConverter.class,
+ category = "verbosity",
+ help = "Use terminal controls to colorize output.")
+ public UseColor useColorEnum;
+
+ @Option(name = "curses",
+ defaultValue = "auto",
+ converter = UseCursesConverter.class,
+ category = "verbosity",
+ help = "Use terminal cursor controls to minimize scrolling output")
+ public UseCurses useCursesEnum;
+
+ @Option(name = "terminal_columns",
+ defaultValue = "80",
+ category = "hidden",
+ help = "A system-generated parameter which specifies the terminal "
+ + " width in columns.")
+ public int terminalColumns;
+
+ @Option(name = "isatty",
+ defaultValue = "false",
+ category = "hidden",
+ help = "A system-generated parameter which is used to notify the "
+ + "server whether this client is running in a terminal. "
+ + "If this is set to false, then '--color=auto' will be treated as '--color=no'. "
+ + "If this is set to true, then '--color=auto' will be treated as '--color=yes'.")
+ public boolean isATty;
+
+ // This lives here (as opposed to the more logical BuildRequest.Options)
+ // because the client passes it to the server *always*. We don't want the
+ // client to have to figure out when it should or shouldn't to send it.
+ @Option(name = "emacs",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "A system-generated parameter which is true iff EMACS=t in the environment of "
+ + "the client. This option controls certain display features.")
+ public boolean runningInEmacs;
+
+ @Option(name = "show_timestamps",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Include timestamps in messages")
+ public boolean showTimestamp;
+
+ @Option(name = "progress_in_terminal_title",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Show the command progress in the terminal title. "
+ + "Useful to see what blaze is doing when having multiple terminal tabs.")
+ public boolean progressInTermTitle;
+
+
+ public boolean useColor() {
+ return useColorEnum == UseColor.YES || (useColorEnum == UseColor.AUTO && isATty);
+ }
+
+ public boolean useCursorControl() {
+ return useCursesEnum == UseCurses.YES || (useCursesEnum == UseCurses.AUTO && isATty);
+ }
+ }
+
+ private static final DateTimeFormatter TIMESTAMP_FORMAT =
+ DateTimeFormat.forPattern("(MM-dd HH:mm:ss.SSS) ");
+
+ protected final OutErr outErr;
+
+ private final PrintStream errPrintStream;
+
+ protected final Set<EventKind> eventMask =
+ EnumSet.copyOf(EventKind.ERRORS_WARNINGS_AND_INFO_AND_OUTPUT);
+
+ protected final boolean showTimestamp;
+
+ public BlazeCommandEventHandler(OutErr outErr, Options eventOptions) {
+ this.outErr = outErr;
+ this.errPrintStream = new PrintStream(outErr.getErrorStream(), true);
+ if (eventOptions.showProgress) {
+ eventMask.add(EventKind.PROGRESS);
+ eventMask.add(EventKind.START);
+ } else {
+ // Skip PASS events if --noshow_progress is requested.
+ eventMask.remove(EventKind.PASS);
+ }
+ if (eventOptions.showTaskFinish) {
+ eventMask.add(EventKind.FINISH);
+ }
+ eventMask.add(EventKind.SUBCOMMAND);
+ this.showTimestamp = eventOptions.showTimestamp;
+ }
+
+ /** See EventHandler.handle. */
+ @Override
+ public void handle(Event event) {
+ if (!eventMask.contains(event.getKind())) {
+ return;
+ }
+ String prefix;
+ switch (event.getKind()) {
+ case STDOUT:
+ putOutput(outErr.getOutputStream(), event);
+ return;
+ case STDERR:
+ putOutput(outErr.getErrorStream(), event);
+ return;
+ case PASS:
+ case FAIL:
+ case TIMEOUT:
+ case ERROR:
+ case WARNING:
+ case DEPCHECKER:
+ prefix = event.getKind() + ": ";
+ break;
+ case SUBCOMMAND:
+ prefix = ">>>>>>>>> ";
+ break;
+ case INFO:
+ case PROGRESS:
+ case START:
+ case FINISH:
+ prefix = "____";
+ break;
+ default:
+ throw new IllegalStateException("" + event.getKind());
+ }
+ StringBuilder buf = new StringBuilder();
+ buf.append(prefix);
+
+ if (showTimestamp) {
+ buf.append(timestamp());
+ }
+
+ Location location = event.getLocation();
+ if (location != null) {
+ buf.append(location.print()).append(": ");
+ }
+
+ buf.append(event.getMessage());
+ if (event.getKind() == EventKind.FINISH) {
+ buf.append(" DONE");
+ }
+
+ // Add a trailing period for ERROR and WARNING messages, which are
+ // typically English sentences composed from exception messages.
+ if (event.getKind() == EventKind.WARNING ||
+ event.getKind() == EventKind.ERROR) {
+ buf.append('.');
+ }
+
+ // Event messages go to stderr; results (e.g. 'blaze query') go to stdout.
+ errPrintStream.println(buf);
+ }
+
+ private void putOutput(OutputStream out, Event event) {
+ try {
+ out.write(event.getMessageBytes());
+ out.flush();
+ } catch (IOException e) {
+ // This can happen in server mode if the blaze client has exited,
+ // or if output is redirected to a file and the disk is full, etc.
+ // Ignore.
+ }
+ }
+
+ /**
+ * @return a string representing the current time, eg "04-26 13:47:32.124".
+ */
+ protected String timestamp() {
+ return TIMESTAMP_FORMAT.print(System.currentTimeMillis());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java
new file mode 100644
index 0000000..ff738db
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java
@@ -0,0 +1,166 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.util.ResourceFileLoader;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Utility class for functionality related to Blaze commands.
+ */
+public class BlazeCommandUtils {
+ /**
+ * Options classes used as startup options in Blaze core.
+ */
+ private static final List<Class<? extends OptionsBase>> DEFAULT_STARTUP_OPTIONS =
+ ImmutableList.<Class<? extends OptionsBase>>of(
+ BlazeServerStartupOptions.class,
+ HostJvmStartupOptions.class);
+
+ /**
+ * The set of option-classes that are common to all Blaze commands.
+ */
+ private static final Collection<Class<? extends OptionsBase>> COMMON_COMMAND_OPTIONS =
+ ImmutableList.of(CommonCommandOptions.class, BlazeCommandEventHandler.Options.class);
+
+
+ private BlazeCommandUtils() {}
+
+ public static ImmutableList<Class<? extends OptionsBase>> getStartupOptions(
+ Iterable<BlazeModule> modules) {
+ Set<Class<? extends OptionsBase>> options = new HashSet<>();
+ options.addAll(DEFAULT_STARTUP_OPTIONS);
+ for (BlazeModule blazeModule : modules) {
+ Iterables.addAll(options, blazeModule.getStartupOptions());
+ }
+
+ return ImmutableList.copyOf(options);
+ }
+
+ /**
+ * Returns the set of all options (including those inherited directly and
+ * transitively) for this AbstractCommand's @Command annotation.
+ *
+ * <p>Why does metaprogramming always seem like such a bright idea in the
+ * beginning?
+ */
+ public static ImmutableList<Class<? extends OptionsBase>> getOptions(
+ Class<? extends BlazeCommand> clazz,
+ Iterable<BlazeModule> modules,
+ ConfiguredRuleClassProvider ruleClassProvider) {
+ Command commandAnnotation = clazz.getAnnotation(Command.class);
+ if (commandAnnotation == null) {
+ throw new IllegalStateException("@Command missing for " + clazz.getName());
+ }
+
+ Set<Class<? extends OptionsBase>> options = new HashSet<>();
+ options.addAll(COMMON_COMMAND_OPTIONS);
+ Collections.addAll(options, commandAnnotation.options());
+
+ if (commandAnnotation.usesConfigurationOptions()) {
+ options.addAll(ruleClassProvider.getConfigurationOptions());
+ }
+
+ for (BlazeModule blazeModule : modules) {
+ Iterables.addAll(options, blazeModule.getCommandOptions(commandAnnotation));
+ }
+
+ for (Class<? extends BlazeCommand> base : commandAnnotation.inherits()) {
+ options.addAll(getOptions(base, modules, ruleClassProvider));
+ }
+ return ImmutableList.copyOf(options);
+ }
+
+ /**
+ * Returns the expansion of the specified help topic.
+ *
+ * @param topic the name of the help topic; used in %{command} expansion.
+ * @param help the text template of the help message. Certain %{x} variables
+ * will be expanded. A prefix of "resource:" means use the .jar
+ * resource of that name.
+ * @param categoryDescriptions a mapping from option category names to
+ * descriptions, passed to {@link OptionsParser#describeOptions}.
+ * @param helpVerbosity a tri-state verbosity option selecting between just
+ * names, names and syntax, and full description.
+ */
+ public static final String expandHelpTopic(String topic, String help,
+ Class<? extends BlazeCommand> commandClass,
+ Collection<Class<? extends OptionsBase>> options,
+ Map<String, String> categoryDescriptions,
+ OptionsParser.HelpVerbosity helpVerbosity) {
+ OptionsParser parser = OptionsParser.newOptionsParser(options);
+
+ String template;
+ if (help.startsWith("resource:")) {
+ String resourceName = help.substring("resource:".length());
+ try {
+ template = ResourceFileLoader.loadResource(commandClass, resourceName);
+ } catch (IOException e) {
+ throw new IllegalStateException("failed to load help resource '" + resourceName
+ + "' due to I/O error: " + e.getMessage(), e);
+ }
+ } else {
+ template = help;
+ }
+
+ if (!template.contains("%{options}")) {
+ throw new IllegalStateException("Help template for '" + topic + "' omits %{options}!");
+ }
+
+ return template.
+ replace("%{command}", topic).
+ replace("%{options}", parser.describeOptions(categoryDescriptions, helpVerbosity)).
+ trim()
+ + "\n\n"
+ + (helpVerbosity == OptionsParser.HelpVerbosity.MEDIUM
+ ? "(Use 'help --long' for full details or --short to just enumerate options.)\n"
+ : "");
+ }
+
+ /**
+ * The help page for this command.
+ *
+ * @param categoryDescriptions a mapping from option category names to
+ * descriptions, passed to {@link OptionsParser#describeOptions}.
+ * @param verbosity a tri-state verbosity option selecting between just names,
+ * names and syntax, and full description.
+ */
+ public static String getUsage(
+ Class<? extends BlazeCommand> commandClass,
+ Map<String, String> categoryDescriptions,
+ OptionsParser.HelpVerbosity verbosity,
+ Iterable<BlazeModule> blazeModules,
+ ConfiguredRuleClassProvider ruleClassProvider) {
+ Command commandAnnotation = commandClass.getAnnotation(Command.class);
+ return BlazeCommandUtils.expandHelpTopic(
+ commandAnnotation.name(),
+ commandAnnotation.help(),
+ commandClass,
+ BlazeCommandUtils.getOptions(commandClass, blazeModules, ruleClassProvider),
+ categoryDescriptions,
+ verbosity);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java
new file mode 100644
index 0000000..6855cbd
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java
@@ -0,0 +1,420 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.actions.ActionContextConsumer;
+import com.google.devtools.build.lib.actions.ActionContextProvider;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.exec.OutputService;
+import com.google.devtools.build.lib.packages.MakeEnvironment;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageFactory.PackageArgument;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+import com.google.devtools.build.lib.query2.output.OutputFormatter;
+import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory;
+import com.google.devtools.build.lib.skyframe.DiffAwareness;
+import com.google.devtools.build.lib.skyframe.PrecomputedValue.Injected;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutorFactory;
+import com.google.devtools.build.lib.syntax.Environment;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import javax.annotation.Nullable;
+
+/**
+ * A module Blaze can load at the beginning of its execution. Modules are supplied with extension
+ * points to augment the functionality at specific, well-defined places.
+ *
+ * <p>The constructors of individual Blaze modules should be empty. All work should be done in the
+ * methods (e.g. {@link #blazeStartup}).
+ */
+public abstract class BlazeModule {
+
+ /**
+ * Returns the extra startup options this module contributes.
+ *
+ * <p>This method will be called at the beginning of Blaze startup (before #blazeStartup).
+ */
+ public Iterable<Class<? extends OptionsBase>> getStartupOptions() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Called before {@link #getFileSystem} and {@link #blazeStartup}.
+ *
+ * <p>This method will be called at the beginning of Blaze startup.
+ */
+ @SuppressWarnings("unused")
+ public void globalInit(OptionsProvider startupOptions) throws AbruptExitException {
+ }
+
+ /**
+ * Returns the file system implementation used by Blaze. It is an error if more than one module
+ * returns a file system. If all return null, the default unix file system is used.
+ *
+ * <p>This method will be called at the beginning of Blaze startup (in-between #globalInit and
+ * #blazeStartup).
+ */
+ @SuppressWarnings("unused")
+ public FileSystem getFileSystem(OptionsProvider startupOptions, PathFragment outputPath) {
+ return null;
+ }
+
+ /**
+ * Called when Blaze starts up.
+ */
+ @SuppressWarnings("unused")
+ public void blazeStartup(OptionsProvider startupOptions,
+ BlazeVersionInfo versionInfo, UUID instanceId, BlazeDirectories directories,
+ Clock clock) throws AbruptExitException {
+ }
+
+ /**
+ * Returns the set of directories under which blaze may assume all files are immutable.
+ */
+ public Set<Path> getImmutableDirectories() {
+ return ImmutableSet.<Path>of();
+ }
+
+ /**
+ * May yield a supplier that provides factories for the Preprocessor to apply. Only one of the
+ * configured modules may return non-null.
+ *
+ * The factory yielded by the supplier will be checked with
+ * {@link Preprocessor.Factory#isStillValid} at the beginning of each incremental build. This
+ * allows modules to have preprocessors customizable by flags.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ public Preprocessor.Factory.Supplier getPreprocessorFactorySupplier() {
+ return null;
+ }
+
+ /**
+ * Adds the rule classes supported by this module.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ @SuppressWarnings("unused")
+ public void initializeRuleClasses(ConfiguredRuleClassProvider.Builder builder) {
+ }
+
+ /**
+ * Returns the list of commands this module contributes to Blaze.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ public Iterable<? extends BlazeCommand> getCommands() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the list of query output formatters this module provides.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ public Iterable<OutputFormatter> getQueryOutputFormatters() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the {@link DiffAwareness} strategies this module contributes. These will be used to
+ * determine which files, if any, changed between Blaze commands.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ @SuppressWarnings("unused")
+ public Iterable<? extends DiffAwareness.Factory> getDiffAwarenessFactories(boolean watchFS) {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the workspace status action factory contributed by this module.
+ *
+ * <p>There should always be exactly one of these in a Blaze instance.
+ */
+ public WorkspaceStatusAction.Factory getWorkspaceStatusActionFactory() {
+ return null;
+ }
+
+ /**
+ * PlatformSet is a group of platforms characterized by a regular expression. For example, the
+ * entry "oldlinux": "i[34]86-libc[345]-linux" might define a set of platforms representing
+ * certain older linux releases.
+ *
+ * <p>Platform-set names are used in BUILD files in the third argument to <tt>vardef</tt>, to
+ * define per-platform tweaks to variables such as CFLAGS.
+ *
+ * <p>vardef is a legacy mechanism: it needs explicit support in the rule implementations,
+ * and cannot express conditional dependencies, only conditional attribute values. This
+ * mechanism will be supplanted by configuration dependent attributes, and its effect can
+ * usually also be achieved with abi_deps.
+ *
+ * <p>This method will be called during Blaze startup (after #blazeStartup).
+ */
+ public Map<String, String> getPlatformSetRegexps() {
+ return ImmutableMap.<String, String>of();
+ }
+
+ /**
+ * Services provided for Blaze modules via BlazeRuntime.
+ */
+ public interface ModuleEnvironment {
+ /**
+ * Gets a file from the depot based on its label and returns the {@link Path} where it can
+ * be found.
+ */
+ Path getFileFromDepot(Label label)
+ throws NoSuchThingException, InterruptedException, IOException;
+
+ /**
+ * Exits Blaze as early as possible. This is currently a hack and should only be called in
+ * event handlers for {@code BuildStartingEvent}, {@code GotOptionsEvent} and
+ * {@code LoadingPhaseCompleteEvent}.
+ */
+ void exit(AbruptExitException exception);
+ }
+
+ /**
+ * Called before each command.
+ */
+ @SuppressWarnings("unused")
+ public void beforeCommand(BlazeRuntime blazeRuntime, Command command)
+ throws AbruptExitException {
+ }
+
+ /**
+ * Returns the output service to be used. It is an error if more than one module returns an
+ * output service.
+ *
+ * <p>This method will be called at the beginning of each command (after #beforeCommand).
+ */
+ @SuppressWarnings("unused")
+ public OutputService getOutputService() throws AbruptExitException {
+ return null;
+ }
+
+ /**
+ * Returns the extra options this module contributes to a specific command.
+ *
+ * <p>This method will be called at the beginning of each command (after #beforeCommand).
+ */
+ @SuppressWarnings("unused")
+ public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns a map of option categories to descriptive strings. This is used by {@code HelpCommand}
+ * to show a more readable list of flags.
+ */
+ public Map<String, String> getOptionCategories() {
+ return ImmutableMap.of();
+ }
+
+ /**
+ * A item that is returned by "blaze info".
+ */
+ public interface InfoItem {
+ /**
+ * The name of the info key.
+ */
+ String getName();
+
+ /**
+ * The help description of the info key.
+ */
+ String getDescription();
+
+ /**
+ * Whether the key is printed when "blaze info" is invoked without arguments.
+ *
+ * <p>This is usually true for info keys that take multiple lines, thus, cannot really be
+ * included in the output of argumentless "blaze info".
+ */
+ boolean isHidden();
+
+ /**
+ * Returns the value of the info key. The return value is directly printed to stdout.
+ */
+ byte[] get(Supplier<BuildConfiguration> configurationSupplier) throws AbruptExitException;
+ }
+
+ /**
+ * Returns the additional information this module provides to "blaze info".
+ *
+ * <p>This method will be called at the beginning of each "blaze info" command (after
+ * #beforeCommand).
+ */
+ public Iterable<InfoItem> getInfoItems() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the list of query functions this module provides to "blaze query".
+ *
+ * <p>This method will be called at the beginning of each "blaze query" command (after
+ * #beforeCommand).
+ */
+ public Iterable<QueryFunction> getQueryFunctions() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Returns the action context provider the module contributes to Blaze, if any.
+ *
+ * <p>This method will be called at the beginning of the execution phase, e.g. of the
+ * "blaze build" command.
+ */
+ public ActionContextProvider getActionContextProvider() {
+ return null;
+ }
+
+ /**
+ * Returns the action context consumer that pulls in action contexts required by this module,
+ * if any.
+ *
+ * <p>This method will be called at the beginning of the execution phase, e.g. of the
+ * "blaze build" command.
+ */
+ public ActionContextConsumer getActionContextConsumer() {
+ return null;
+ }
+
+ /**
+ * Called after each command.
+ */
+ public void afterCommand() {
+ }
+
+ /**
+ * Called when Blaze shuts down.
+ */
+ public void blazeShutdown() {
+ }
+
+ /**
+ * Action inputs are allowed to be missing for all inputs where this predicate returns true.
+ */
+ public Predicate<PathFragment> getAllowedMissingInputs() {
+ return null;
+ }
+
+ /**
+ * Optionally specializes the cache that ensures source files are looked at just once during
+ * a build. Only one module may do so.
+ */
+ public ActionInputFileCache createActionInputCache(String cwd, FileSystem fs) {
+ return null;
+ }
+
+ /**
+ * Returns the extensions this module contributes to the global namespace of the BUILD language.
+ */
+ public PackageFactory.EnvironmentExtension getPackageEnvironmentExtension() {
+ return new PackageFactory.EnvironmentExtension() {
+ @Override
+ public void update(
+ Environment environment, MakeEnvironment.Builder pkgMakeEnv, Label buildFileLabel) {
+ }
+
+ @Override
+ public Iterable<PackageArgument<?>> getPackageArguments() {
+ return ImmutableList.of();
+ }
+ };
+ }
+
+ /**
+ * Returns a factory for creating {@link SkyframeExecutor} objects. If the module does not
+ * provide any SkyframeExecutorFactory, it returns null. Note that only one factory per
+ * Bazel/Blaze runtime is allowed.
+ */
+ public SkyframeExecutorFactory getSkyframeExecutorFactory() {
+ return null;
+ }
+
+ /** Returns a map of "extra" SkyFunctions for SkyValues that this module may want to build. */
+ public ImmutableMap<SkyFunctionName, SkyFunction> getSkyFunctions(BlazeDirectories directories) {
+ return ImmutableMap.of();
+ }
+
+ /**
+ * Returns the extra precomputed values that the module makes available in Skyframe.
+ *
+ * <p>This method is called once per Blaze instance at the very beginning of its life.
+ * If it creates the injected values by using a {@code com.google.common.base.Supplier},
+ * that supplier is asked for the value it contains just before the loading phase begins. This
+ * functionality can be used to implement precomputed values that are not constant during the
+ * lifetime of a Blaze instance (naturally, they must be constant over the course of a build)
+ *
+ * <p>The following things must be done in order to define a new precomputed values:
+ * <ul>
+ * <li> Create a public static final variable of type
+ * {@link com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed}
+ * <li> Set its value by adding an {@link Injected} in this method (it can be created using the
+ * aforementioned variable and the value or a supplier of the value)
+ * <li> Reference the value in Skyframe functions by calling get {@code get} method on the
+ * {@link com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed} variable. This
+ * will never return null, because its value will have been injected before most of the Skyframe
+ * values are computed.
+ * </ul>
+ */
+ public Iterable<Injected> getPrecomputedSkyframeValues() {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Optionally returns a provider for project files that can be used to bundle targets and
+ * command-line options.
+ */
+ @Nullable
+ public ProjectFile.Provider createProjectFileProvider() {
+ return null;
+ }
+
+ /**
+ * Optionally returns a factory to create coverage report actions.
+ */
+ @Nullable
+ public CoverageReportActionFactory getCoverageReportFactory() {
+ return null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
new file mode 100644
index 0000000..0251e83
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
@@ -0,0 +1,1795 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.SubscriberExceptionContext;
+import com.google.common.eventbus.SubscriberExceptionHandler;
+import com.google.common.util.concurrent.Uninterruptibles;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.actions.cache.ActionCache;
+import com.google.devtools.build.lib.actions.cache.CompactPersistentActionCache;
+import com.google.devtools.build.lib.actions.cache.NullActionCache;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.analysis.BuildView;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
+import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
+import com.google.devtools.build.lib.analysis.config.DefaultsPackage;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.buildtool.BuildTool;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.OutputFilter;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.exec.OutputService;
+import com.google.devtools.build.lib.packages.NoSuchThingException;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider;
+import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.pkgcache.PackageManager;
+import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
+import com.google.devtools.build.lib.profiler.MemoryProfiler;
+import com.google.devtools.build.lib.profiler.ProfilePhase;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.Profiler.ProfiledTaskKinds;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.query2.output.OutputFormatter;
+import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory;
+import com.google.devtools.build.lib.runtime.commands.BuildCommand;
+import com.google.devtools.build.lib.runtime.commands.CanonicalizeCommand;
+import com.google.devtools.build.lib.runtime.commands.CleanCommand;
+import com.google.devtools.build.lib.runtime.commands.HelpCommand;
+import com.google.devtools.build.lib.runtime.commands.InfoCommand;
+import com.google.devtools.build.lib.runtime.commands.ProfileCommand;
+import com.google.devtools.build.lib.runtime.commands.QueryCommand;
+import com.google.devtools.build.lib.runtime.commands.RunCommand;
+import com.google.devtools.build.lib.runtime.commands.ShutdownCommand;
+import com.google.devtools.build.lib.runtime.commands.SkylarkCommand;
+import com.google.devtools.build.lib.runtime.commands.TestCommand;
+import com.google.devtools.build.lib.runtime.commands.VersionCommand;
+import com.google.devtools.build.lib.server.RPCServer;
+import com.google.devtools.build.lib.server.ServerCommand;
+import com.google.devtools.build.lib.server.signal.InterruptSignalHandler;
+import com.google.devtools.build.lib.skyframe.DiffAwareness;
+import com.google.devtools.build.lib.skyframe.PrecomputedValue;
+import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutorFactory;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.skyframe.SkyframeExecutorFactory;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Clock;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.util.OsUtils;
+import com.google.devtools.build.lib.util.ThreadUtils;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.JavaIoFileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.UnixFileSystem;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionPriority;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsClassProvider;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.devtools.common.options.OptionsProvider;
+import com.google.devtools.common.options.TriState;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nullable;
+
+/**
+ * The BlazeRuntime class encapsulates the runtime settings and services that
+ * are available to most parts of any Blaze application for the duration of the
+ * batch run or server lifetime. A single instance of this runtime will exist
+ * and will be passed around as needed.
+ */
+public final class BlazeRuntime {
+ /**
+ * The threshold for memory reserved by a 32-bit JVM before trouble may be expected.
+ *
+ * <p>After the JVM starts, it reserves memory for heap (controlled by -Xmx) and non-heap
+ * (code, PermGen, etc.). Furthermore, as Blaze spawns threads, each thread reserves memory
+ * for the stack (controlled by -Xss). Thus even if Blaze starts fine, with high memory settings
+ * it will die from a stack allocation failure in the middle of a build. We prefer failing
+ * upfront by setting a safe threshold.
+ *
+ * <p>This does not apply to 64-bit VMs.
+ */
+ private static final long MAX_BLAZE32_RESERVED_MEMORY = 3400 * 1048576L;
+
+ // Less than this indicates tampering with -Xmx settings.
+ private static final long MIN_BLAZE32_HEAP_SIZE = 3000 * 1000000L;
+
+ public static final String DO_NOT_BUILD_FILE_NAME = "DO_NOT_BUILD_HERE";
+
+ private static final Pattern suppressFromLog = Pattern.compile(".*(auth|pass|cookie).*",
+ Pattern.CASE_INSENSITIVE);
+
+ private static final Logger LOG = Logger.getLogger(BlazeRuntime.class.getName());
+
+ private final BlazeDirectories directories;
+ private Path workingDirectory;
+ private long commandStartTime;
+
+ // Application-specified constants
+ private final PathFragment runfilesPrefix;
+
+ private final SkyframeExecutor skyframeExecutor;
+
+ private final Reporter reporter;
+ private EventBus eventBus;
+ private final LoadingPhaseRunner loadingPhaseRunner;
+ private final PackageFactory packageFactory;
+ private final ConfigurationFactory configurationFactory;
+ private final ConfiguredRuleClassProvider ruleClassProvider;
+ private final BuildView view;
+ private ActionCache actionCache;
+ private final TimestampGranularityMonitor timestampGranularityMonitor;
+ private final Clock clock;
+ private final BuildTool buildTool;
+
+ private OutputService outputService;
+
+ private final Iterable<BlazeModule> blazeModules;
+ private final BlazeModule.ModuleEnvironment blazeModuleEnvironment;
+
+ private UUID commandId; // Unique identifier for the command being run
+
+ private final AtomicInteger storedExitCode = new AtomicInteger();
+
+ private final Map<String, String> clientEnv;
+
+ // We pass this through here to make it available to the MasterLogWriter.
+ private final OptionsProvider startupOptionsProvider;
+
+ private String outputFileSystem;
+ private Map<String, BlazeCommand> commandMap;
+
+ private AbruptExitException pendingException;
+
+ private final SubscriberExceptionHandler eventBusExceptionHandler;
+
+ private final BinTools binTools;
+
+ private final WorkspaceStatusAction.Factory workspaceStatusActionFactory;
+
+ private final ProjectFile.Provider projectFileProvider;
+
+ private class BlazeModuleEnvironment implements BlazeModule.ModuleEnvironment {
+ @Override
+ public Path getFileFromDepot(Label label)
+ throws NoSuchThingException, InterruptedException, IOException {
+ Target target = getPackageManager().getTarget(reporter, label);
+ return (outputService != null)
+ ? outputService.stageTool(target)
+ : target.getPackage().getPackageDirectory().getRelative(target.getName());
+ }
+
+ @Override
+ public void exit(AbruptExitException exception) {
+ Preconditions.checkState(pendingException == null);
+ pendingException = exception;
+ }
+ }
+
+ private BlazeRuntime(BlazeDirectories directories, Reporter reporter,
+ WorkspaceStatusAction.Factory workspaceStatusActionFactory,
+ final SkyframeExecutor skyframeExecutor,
+ PackageFactory pkgFactory, ConfiguredRuleClassProvider ruleClassProvider,
+ ConfigurationFactory configurationFactory, PathFragment runfilesPrefix, Clock clock,
+ OptionsProvider startupOptionsProvider, Iterable<BlazeModule> blazeModules,
+ Map<String, String> clientEnv,
+ TimestampGranularityMonitor timestampGranularityMonitor,
+ SubscriberExceptionHandler eventBusExceptionHandler,
+ BinTools binTools, ProjectFile.Provider projectFileProvider) {
+ this.workspaceStatusActionFactory = workspaceStatusActionFactory;
+ this.directories = directories;
+ this.workingDirectory = directories.getWorkspace();
+ this.reporter = reporter;
+ this.runfilesPrefix = runfilesPrefix;
+ this.packageFactory = pkgFactory;
+ this.binTools = binTools;
+ this.projectFileProvider = projectFileProvider;
+
+ this.skyframeExecutor = skyframeExecutor;
+ this.loadingPhaseRunner = new LoadingPhaseRunner(
+ skyframeExecutor.getPackageManager(),
+ pkgFactory.getRuleClassNames());
+
+ this.clientEnv = clientEnv;
+
+ this.blazeModules = blazeModules;
+ this.ruleClassProvider = ruleClassProvider;
+ this.configurationFactory = configurationFactory;
+ this.view = new BuildView(directories, getPackageManager(), ruleClassProvider,
+ skyframeExecutor, binTools, getCoverageReportActionFactory(blazeModules));
+ this.clock = clock;
+ this.timestampGranularityMonitor = Preconditions.checkNotNull(timestampGranularityMonitor);
+ this.startupOptionsProvider = startupOptionsProvider;
+
+ this.eventBusExceptionHandler = eventBusExceptionHandler;
+ this.blazeModuleEnvironment = new BlazeModuleEnvironment();
+ this.buildTool = new BuildTool(this);
+ initEventBus();
+
+ if (inWorkspace()) {
+ writeOutputBaseReadmeFile();
+ writeOutputBaseDoNotBuildHereFile();
+ }
+ setupExecRoot();
+ }
+
+ @Nullable private CoverageReportActionFactory getCoverageReportActionFactory(
+ Iterable<BlazeModule> blazeModules) {
+ CoverageReportActionFactory firstFactory = null;
+ for (BlazeModule module : blazeModules) {
+ CoverageReportActionFactory factory = module.getCoverageReportFactory();
+ if (factory != null) {
+ Preconditions.checkState(firstFactory == null,
+ "only one Blaze Module can have a Coverage Report Factory");
+ firstFactory = factory;
+ }
+ }
+ return firstFactory;
+ }
+
+ /**
+ * Figures out what file system we are writing output to. Here we use
+ * outputBase instead of outputPath because we need a file system to create the latter.
+ */
+ private String determineOutputFileSystem() {
+ if (getOutputService() != null) {
+ return getOutputService().getFilesSystemName();
+ }
+ long startTime = Profiler.nanoTimeMaybe();
+ String fileSystem = FileSystemUtils.getFileSystem(getOutputBase());
+ Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, "Finding output file system");
+ return fileSystem;
+ }
+
+ public String getOutputFileSystem() {
+ return outputFileSystem;
+ }
+
+ @VisibleForTesting
+ public void initEventBus() {
+ setEventBus(new EventBus(eventBusExceptionHandler));
+ }
+
+ private void clearEventBus() {
+ // EventBus does not have an unregister() method, so this is how we release memory associated
+ // with handlers.
+ setEventBus(null);
+ }
+
+ private void setEventBus(EventBus eventBus) {
+ this.eventBus = eventBus;
+ skyframeExecutor.setEventBus(eventBus);
+ }
+
+ /**
+ * Conditionally enable profiling.
+ */
+ private final boolean initProfiler(CommonCommandOptions options,
+ UUID buildID, long execStartTimeNanos) {
+ OutputStream out = null;
+ boolean recordFullProfilerData = false;
+ ProfiledTaskKinds profiledTasks = ProfiledTaskKinds.NONE;
+
+ try {
+ if (options.profilePath != null) {
+ Path profilePath = getWorkspace().getRelative(options.profilePath);
+
+ recordFullProfilerData = options.recordFullProfilerData;
+ out = new BufferedOutputStream(profilePath.getOutputStream(), 1024 * 1024);
+ getReporter().handle(Event.info("Writing profile data to '" + profilePath + "'"));
+ profiledTasks = ProfiledTaskKinds.ALL;
+ } else if (options.alwaysProfileSlowOperations) {
+ recordFullProfilerData = false;
+ out = null;
+ profiledTasks = ProfiledTaskKinds.SLOWEST;
+ }
+ if (profiledTasks != ProfiledTaskKinds.NONE) {
+ Profiler.instance().start(profiledTasks, out,
+ "Blaze profile for " + getOutputBase() + " at " + new Date()
+ + ", build ID: " + buildID,
+ recordFullProfilerData, clock, execStartTimeNanos);
+ return true;
+ }
+ } catch (IOException e) {
+ getReporter().handle(Event.error("Error while creating profile file: " + e.getMessage()));
+ }
+ return false;
+ }
+
+ /**
+ * Generates a README file in the output base directory. This README file
+ * contains the name of the workspace directory, so that users can figure out
+ * which output base directory corresponds to which workspace.
+ */
+ private void writeOutputBaseReadmeFile() {
+ Preconditions.checkNotNull(getWorkspace());
+ Path outputBaseReadmeFile = getOutputBase().getRelative("README");
+ try {
+ FileSystemUtils.writeIsoLatin1(outputBaseReadmeFile, "WORKSPACE: " + getWorkspace(), "",
+ "The first line of this file is intentionally easy to parse for various",
+ "interactive scripting and debugging purposes. But please DO NOT write programs",
+ "that exploit it, as they will be broken by design: it is not possible to",
+ "reverse engineer the set of source trees or the --package_path from the output",
+ "tree, and if you attempt it, you will fail, creating subtle and",
+ "hard-to-diagnose bugs, that will no doubt get blamed on changes made by the",
+ "Blaze team.", "", "This directory was generated by Blaze.",
+ "Do not attempt to modify or delete any files in this directory.",
+ "Among other issues, Blaze's file system caching assumes that",
+ "only Blaze will modify this directory and the files in it,",
+ "so if you change anything here you may mess up Blaze's cache.");
+ } catch (IOException e) {
+ LOG.warning("Couldn't write to '" + outputBaseReadmeFile + "': " + e.getMessage());
+ }
+ }
+
+ private void writeOutputBaseDoNotBuildHereFile() {
+ Preconditions.checkNotNull(getWorkspace());
+ Path filePath = getOutputBase().getRelative(DO_NOT_BUILD_FILE_NAME);
+ try {
+ FileSystemUtils.writeContent(filePath, ISO_8859_1, getWorkspace().toString());
+ } catch (IOException e) {
+ LOG.warning("Couldn't write to '" + filePath + "': " + e.getMessage());
+ }
+ }
+
+ /**
+ * Creates the execRoot dir under outputBase.
+ */
+ private void setupExecRoot() {
+ try {
+ FileSystemUtils.createDirectoryAndParents(directories.getExecRoot());
+ } catch (IOException e) {
+ LOG.warning("failed to create execution root '" + directories.getExecRoot() + "': "
+ + e.getMessage());
+ }
+ }
+
+ public void recordCommandStartTime(long commandStartTime) {
+ this.commandStartTime = commandStartTime;
+ }
+
+ public long getCommandStartTime() {
+ return commandStartTime;
+ }
+
+ public String getWorkspaceName() {
+ Path workspace = directories.getWorkspace();
+ if (workspace == null) {
+ return "";
+ }
+ return workspace.getBaseName();
+ }
+
+ /**
+ * Returns any prefix to be inserted between relative source paths and the runfiles directory.
+ */
+ public PathFragment getRunfilesPrefix() {
+ return runfilesPrefix;
+ }
+
+ /**
+ * Returns the Blaze directories object for this runtime.
+ */
+ public BlazeDirectories getDirectories() {
+ return directories;
+ }
+
+ /**
+ * Returns the working directory of the server.
+ *
+ * <p>This is often the first entry on the {@code --package_path}, but not always.
+ * Callers should certainly not make this assumption. The Path returned may be null.
+ *
+ * @see #getWorkingDirectory()
+ */
+ public Path getWorkspace() {
+ return directories.getWorkspace();
+ }
+
+ /**
+ * Returns the working directory of the {@code blaze} client process.
+ *
+ * <p>This may be equal to {@code getWorkspace()}, or beneath it.
+ *
+ * @see #getWorkspace()
+ */
+ public Path getWorkingDirectory() {
+ return workingDirectory;
+ }
+
+ /**
+ * Returns if the client passed a valid workspace to be used for the build.
+ */
+ public boolean inWorkspace() {
+ return directories.inWorkspace();
+ }
+
+ /**
+ * Returns the output base directory associated with this Blaze server
+ * process. This is the base directory for shared Blaze state as well as tool
+ * and strategy specific subdirectories.
+ */
+ public Path getOutputBase() {
+ return directories.getOutputBase();
+ }
+
+ /**
+ * Returns the output path associated with this Blaze server process..
+ */
+ public Path getOutputPath() {
+ return directories.getOutputPath();
+ }
+
+ /**
+ * The directory in which blaze stores the server state - that is, the socket
+ * file and a log.
+ */
+ public Path getServerDirectory() {
+ return getOutputBase().getChild("server");
+ }
+
+ /**
+ * Returns the execution root directory associated with this Blaze server
+ * process. This is where all input and output files visible to the actual
+ * build reside.
+ */
+ public Path getExecRoot() {
+ return directories.getExecRoot();
+ }
+
+ /**
+ * Returns the reporter for events.
+ */
+ public Reporter getReporter() {
+ return reporter;
+ }
+
+ /**
+ * Returns the current event bus. Only valid within the scope of a single Blaze command.
+ */
+ public EventBus getEventBus() {
+ return eventBus;
+ }
+
+ public BinTools getBinTools() {
+ return binTools;
+ }
+
+ /**
+ * Returns the skyframe executor.
+ */
+ public SkyframeExecutor getSkyframeExecutor() {
+ return skyframeExecutor;
+ }
+
+ /**
+ * Returns the package factory.
+ */
+ public PackageFactory getPackageFactory() {
+ return packageFactory;
+ }
+
+ /**
+ * Returns the build tool.
+ */
+ public BuildTool getBuildTool() {
+ return buildTool;
+ }
+
+ public ImmutableList<OutputFormatter> getQueryOutputFormatters() {
+ ImmutableList.Builder<OutputFormatter> result = ImmutableList.builder();
+ result.addAll(OutputFormatter.getDefaultFormatters());
+ for (BlazeModule module : blazeModules) {
+ result.addAll(module.getQueryOutputFormatters());
+ }
+
+ return result.build();
+ }
+
+ /**
+ * Returns the package manager.
+ */
+ public PackageManager getPackageManager() {
+ return skyframeExecutor.getPackageManager();
+ }
+
+ public WorkspaceStatusAction.Factory getworkspaceStatusActionFactory() {
+ return workspaceStatusActionFactory;
+ }
+
+ public BlazeModule.ModuleEnvironment getBlazeModuleEnvironment() {
+ return blazeModuleEnvironment;
+ }
+
+ /**
+ * Returns the rule class provider.
+ */
+ public ConfiguredRuleClassProvider getRuleClassProvider() {
+ return ruleClassProvider;
+ }
+
+ public LoadingPhaseRunner getLoadingPhaseRunner() {
+ return loadingPhaseRunner;
+ }
+
+ /**
+ * Returns the build view.
+ */
+ public BuildView getView() {
+ return view;
+ }
+
+ public Iterable<BlazeModule> getBlazeModules() {
+ return blazeModules;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T extends BlazeModule> T getBlazeModule(Class<T> moduleClass) {
+ for (BlazeModule module : blazeModules) {
+ if (module.getClass() == moduleClass) {
+ return (T) module;
+ }
+ }
+
+ return null;
+ }
+
+ public ConfigurationFactory getConfigurationFactory() {
+ return configurationFactory;
+ }
+
+ /**
+ * Returns the target pattern parser.
+ */
+ public TargetPatternEvaluator getTargetPatternEvaluator() {
+ return loadingPhaseRunner.getTargetPatternEvaluator();
+ }
+
+ /**
+ * Returns reference to the lazily instantiated persistent action cache
+ * instance. Note, that method may recreate instance between different build
+ * requests, so return value should not be cached.
+ */
+ public ActionCache getPersistentActionCache() throws IOException {
+ if (actionCache == null) {
+ if (OS.getCurrent() == OS.WINDOWS) {
+ // TODO(bazel-team): Add support for a persistent action cache on Windows.
+ actionCache = new NullActionCache();
+ return actionCache;
+ }
+ long startTime = Profiler.nanoTimeMaybe();
+ try {
+ actionCache = new CompactPersistentActionCache(getCacheDirectory(), clock);
+ } catch (IOException e) {
+ LOG.log(Level.WARNING, "Failed to load action cache: " + e.getMessage(), e);
+ LoggingUtil.logToRemote(Level.WARNING, "Failed to load action cache: "
+ + e.getMessage(), e);
+ getReporter().handle(
+ Event.error("Error during action cache initialization: " + e.getMessage()
+ + ". Corrupted files were renamed to '" + getCacheDirectory() + "/*.bad'. "
+ + "Blaze will now reset action cache data, causing a full rebuild"));
+ actionCache = new CompactPersistentActionCache(getCacheDirectory(), clock);
+ } finally {
+ Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, "Loading action cache");
+ }
+ }
+ return actionCache;
+ }
+
+ /**
+ * Removes in-memory caches.
+ */
+ public void clearCaches() throws IOException {
+ clearSkyframeRelevantCaches();
+ actionCache = null;
+ FileSystemUtils.deleteTree(getCacheDirectory());
+ }
+
+ /** Removes skyframe cache and other caches that must be kept synchronized with skyframe. */
+ private void clearSkyframeRelevantCaches() {
+ skyframeExecutor.resetEvaluator();
+ view.clear();
+ }
+
+ /**
+ * Returns the TimestampGranularityMonitor. The same monitor object is used
+ * across multiple Blaze commands, but it doesn't hold any persistent state
+ * across different commands.
+ */
+ public TimestampGranularityMonitor getTimestampGranularityMonitor() {
+ return timestampGranularityMonitor;
+ }
+
+ /**
+ * Returns path to the cache directory. Path must be inside output base to
+ * ensure that users can run concurrent instances of blaze in different
+ * clients without attempting to concurrently write to the same action cache
+ * on disk, which might not be safe.
+ */
+ private Path getCacheDirectory() {
+ return getOutputBase().getChild("action_cache");
+ }
+
+ /**
+ * Returns a provider for project file objects. Can be null if no such provider was set by any of
+ * the modules.
+ */
+ @Nullable
+ public ProjectFile.Provider getProjectFileProvider() {
+ return projectFileProvider;
+ }
+
+ /**
+ * Hook method called by the BlazeCommandDispatcher prior to the dispatch of
+ * each command.
+ *
+ * @param options The CommonCommandOptions used by every command.
+ * @throws AbruptExitException if this command is unsuitable to be run as specified
+ */
+ void beforeCommand(String commandName, OptionsParser optionsParser,
+ CommonCommandOptions options, long execStartTimeNanos)
+ throws AbruptExitException {
+ commandStartTime -= options.startupTime;
+
+ eventBus.post(new GotOptionsEvent(startupOptionsProvider,
+ optionsParser));
+ throwPendingException();
+
+ outputService = null;
+ BlazeModule outputModule = null;
+ for (BlazeModule module : blazeModules) {
+ OutputService moduleService = module.getOutputService();
+ if (moduleService != null) {
+ if (outputService != null) {
+ throw new IllegalStateException(String.format(
+ "More than one module (%s and %s) returns an output service",
+ module.getClass(), outputModule.getClass()));
+ }
+ outputService = moduleService;
+ outputModule = module;
+ }
+ }
+
+ skyframeExecutor.setBatchStatter(outputService == null
+ ? null
+ : outputService.getBatchStatter());
+
+ outputFileSystem = determineOutputFileSystem();
+
+ // Ensure that the working directory will be under the workspace directory.
+ Path workspace = getWorkspace();
+ if (inWorkspace()) {
+ workingDirectory = workspace.getRelative(options.clientCwd);
+ } else {
+ workspace = FileSystemUtils.getWorkingDirectory(directories.getFileSystem());
+ workingDirectory = workspace;
+ }
+ updateClientEnv(options.clientEnv, options.ignoreClientEnv);
+ loadingPhaseRunner.updatePatternEvaluator(workingDirectory.relativeTo(workspace));
+
+ // Fail fast in the case where a Blaze command forgets to install the package path correctly.
+ skyframeExecutor.setActive(false);
+ // Let skyframe figure out if it needs to store graph edges for this build.
+ skyframeExecutor.decideKeepIncrementalState(
+ startupOptionsProvider.getOptions(BlazeServerStartupOptions.class).batch,
+ optionsParser.getOptions(BuildView.Options.class));
+
+ // Conditionally enable profiling
+ // We need to compensate for launchTimeNanos (measurements taken outside of the jvm).
+ long startupTimeNanos = options.startupTime * 1000000L;
+ if (initProfiler(options, this.getCommandId(), execStartTimeNanos - startupTimeNanos)) {
+ Profiler profiler = Profiler.instance();
+
+ // Instead of logEvent() we're calling the low level function to pass the timings we took in
+ // the launcher. We're setting the INIT phase marker so that it follows immediately the LAUNCH
+ // phase.
+ profiler.logSimpleTaskDuration(execStartTimeNanos - startupTimeNanos, 0, ProfilerTask.PHASE,
+ ProfilePhase.LAUNCH.description);
+ profiler.logSimpleTaskDuration(execStartTimeNanos, 0, ProfilerTask.PHASE,
+ ProfilePhase.INIT.description);
+ }
+
+ if (options.memoryProfilePath != null) {
+ Path memoryProfilePath = getWorkingDirectory().getRelative(options.memoryProfilePath);
+ try {
+ MemoryProfiler.instance().start(memoryProfilePath.getOutputStream());
+ } catch (IOException e) {
+ getReporter().handle(
+ Event.error("Error while creating memory profile file: " + e.getMessage()));
+ }
+ }
+
+ eventBus.post(new CommandStartEvent(commandName, commandId, clientEnv, workingDirectory));
+ // Initialize exit code to dummy value for afterCommand.
+ storedExitCode.set(ExitCode.RESERVED.getNumericExitCode());
+ }
+
+ /**
+ * Hook method called by the BlazeCommandDispatcher right before the dispatch
+ * of each command ends (while its outcome can still be modified).
+ */
+ ExitCode precompleteCommand(ExitCode originalExit) {
+ eventBus.post(new CommandPrecompleteEvent(originalExit));
+ // If Blaze did not suffer an infrastructure failure, check for errors in modules.
+ ExitCode exitCode = originalExit;
+ if (!originalExit.isInfrastructureFailure()) {
+ if (pendingException != null) {
+ exitCode = pendingException.getExitCode();
+ }
+ }
+ pendingException = null;
+ return exitCode;
+ }
+
+ /**
+ * Posts the {@link CommandCompleteEvent}, so that listeners can tidy up. Called by {@link
+ * #afterCommand}, and by BugReport when crashing from an exception in an async thread.
+ */
+ public void notifyCommandComplete(int exitCode) {
+ if (!storedExitCode.compareAndSet(ExitCode.RESERVED.getNumericExitCode(), exitCode)) {
+ // This command has already been called, presumably because there is a race between the main
+ // thread and a worker thread that crashed. Don't try to arbitrate the dispute. If the main
+ // thread won the race (unlikely, but possible), this may be incorrectly logged as a success.
+ return;
+ }
+ eventBus.post(new CommandCompleteEvent(exitCode));
+ }
+
+ /**
+ * Hook method called by the BlazeCommandDispatcher after the dispatch of each
+ * command.
+ */
+ @VisibleForTesting
+ public void afterCommand(int exitCode) {
+ // Remove any filters that the command might have added to the reporter.
+ getReporter().setOutputFilter(OutputFilter.OUTPUT_EVERYTHING);
+
+ notifyCommandComplete(exitCode);
+
+ for (BlazeModule module : blazeModules) {
+ module.afterCommand();
+ }
+
+ clearEventBus();
+
+ try {
+ Profiler.instance().stop();
+ MemoryProfiler.instance().stop();
+ } catch (IOException e) {
+ getReporter().handle(Event.error("Error while writing profile file: " + e.getMessage()));
+ }
+ }
+
+ // Make sure we keep a strong reference to this logger, so that the
+ // configuration isn't lost when the gc kicks in.
+ private static Logger templateLogger = Logger.getLogger("com.google.devtools.build");
+
+ /**
+ * Configures "com.google.devtools.build.*" loggers to the given
+ * {@code level}. Note: This code relies on static state.
+ */
+ public static void setupLogging(Level level) {
+ templateLogger.setLevel(level);
+ templateLogger.info("Log level: " + templateLogger.getLevel());
+ }
+
+ /**
+ * Return an unmodifiable view of the blaze client's environment when it
+ * invoked the most recent command. Updates from future requests will be
+ * accessible from this view.
+ */
+ public Map<String, String> getClientEnv() {
+ return Collections.unmodifiableMap(clientEnv);
+ }
+
+ @VisibleForTesting
+ void updateClientEnv(List<Map.Entry<String, String>> clientEnvList, boolean ignoreClientEnv) {
+ clientEnv.clear();
+
+ Collection<Map.Entry<String, String>> env =
+ ignoreClientEnv ? System.getenv().entrySet() : clientEnvList;
+ for (Map.Entry<String, String> entry : env) {
+ clientEnv.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Returns the Clock-instance used for the entire build. Before,
+ * individual classes (such as Profiler) used to specify the type
+ * of clock (e.g. EpochClock) they wanted to use. This made it
+ * difficult to get Blaze working on Windows as some of the clocks
+ * available for Linux aren't (directly) available on Windows.
+ * Setting the Blaze-wide clock upon construction of BlazeRuntime
+ * allows injecting whatever Clock instance should be used from
+ * BlazeMain.
+ *
+ * @return The Blaze-wide clock
+ */
+ public Clock getClock() {
+ return clock;
+ }
+
+ public OptionsProvider getStartupOptionsProvider() {
+ return startupOptionsProvider;
+ }
+
+ /**
+ * An array of String values useful if Blaze crashes.
+ * For now, just returns the size of the action cache and the build id.
+ */
+ public String[] getCrashData() {
+ return new String[]{
+ getFileSizeString(CompactPersistentActionCache.cacheFile(getCacheDirectory()),
+ "action cache"),
+ commandIdString(),
+ };
+ }
+
+ private String commandIdString() {
+ UUID uuid = getCommandId();
+ return (uuid == null)
+ ? "no build id"
+ : uuid + " (build id)";
+ }
+
+ /**
+ * @return the OutputService in use, or null if none.
+ */
+ public OutputService getOutputService() {
+ return outputService;
+ }
+
+ private String getFileSizeString(Path path, String type) {
+ try {
+ return String.format("%d bytes (%s)", path.getFileSize(), type);
+ } catch (IOException e) {
+ return String.format("unknown file size (%s)", type);
+ }
+ }
+
+ /**
+ * Returns the UUID that Blaze uses to identify everything
+ * logged from the current build command.
+ */
+ public UUID getCommandId() {
+ return commandId;
+ }
+
+ void setCommandMap(Map<String, BlazeCommand> commandMap) {
+ this.commandMap = ImmutableMap.copyOf(commandMap);
+ }
+
+ public Map<String, BlazeCommand> getCommandMap() {
+ return commandMap;
+ }
+
+ /**
+ * Sets the UUID that Blaze uses to identify everything
+ * logged from the current build command.
+ */
+ @VisibleForTesting
+ public void setCommandId(UUID runId) {
+ commandId = runId;
+ }
+
+ /**
+ * Constructs a build configuration key for the given options.
+ */
+ public BuildConfigurationKey getBuildConfigurationKey(BuildOptions buildOptions,
+ ImmutableSortedSet<String> multiCpu) {
+ return new BuildConfigurationKey(buildOptions, directories, clientEnv, multiCpu);
+ }
+
+ /**
+ * This method only exists for the benefit of InfoCommand, which needs to construct a {@link
+ * BuildConfigurationCollection} without running a full loading phase. Don't add any more clients;
+ * instead, we should change info so that it doesn't need the configuration.
+ */
+ public BuildConfigurationCollection getConfigurations(OptionsProvider optionsProvider)
+ throws InvalidConfigurationException, InterruptedException {
+ BuildConfigurationKey configurationKey = getBuildConfigurationKey(
+ createBuildOptions(optionsProvider), ImmutableSortedSet.<String>of());
+ boolean keepGoing = optionsProvider.getOptions(BuildView.Options.class).keepGoing;
+ LoadedPackageProvider loadedPackageProvider =
+ loadingPhaseRunner.loadForConfigurations(reporter,
+ ImmutableSet.copyOf(configurationKey.getLabelsToLoadUnconditionally().values()),
+ keepGoing);
+ if (loadedPackageProvider == null) {
+ throw new InvalidConfigurationException("Configuration creation failed");
+ }
+ return skyframeExecutor.createConfigurations(keepGoing, configurationFactory,
+ configurationKey);
+ }
+
+ /**
+ * Initializes the package cache using the given options, and syncs the package cache. Also
+ * injects a defaults package using the options for the {@link BuildConfiguration}.
+ *
+ * @see DefaultsPackage
+ */
+ public void setupPackageCache(PackageCacheOptions packageCacheOptions,
+ String defaultsPackageContents) throws InterruptedException, AbruptExitException {
+ if (!skyframeExecutor.hasIncrementalState()) {
+ clearSkyframeRelevantCaches();
+ }
+ skyframeExecutor.sync(packageCacheOptions, getWorkingDirectory(),
+ defaultsPackageContents, getCommandId());
+ }
+
+ public void shutdown() {
+ for (BlazeModule module : blazeModules) {
+ module.blazeShutdown();
+ }
+ }
+
+ /**
+ * Throws the exception currently queued by a Blaze module.
+ *
+ * <p>This should be called as often as is practical so that errors are reported as soon as
+ * possible. Ideally, we'd not need this, but the event bus swallows exceptions so we raise
+ * the exception this way.
+ */
+ public void throwPendingException() throws AbruptExitException {
+ if (pendingException != null) {
+ AbruptExitException exception = pendingException;
+ pendingException = null;
+ throw exception;
+ }
+ }
+
+ /**
+ * Returns the defaults package for the default settings. Should only be called by commands that
+ * do <i>not</i> process {@link BuildOptions}, since build options can alter the contents of the
+ * defaults package, which will not be reflected here.
+ */
+ public String getDefaultsPackageContent() {
+ return ruleClassProvider.getDefaultsPackageContent();
+ }
+
+ /**
+ * Returns the defaults package for the given options taken from an optionsProvider.
+ */
+ public String getDefaultsPackageContent(OptionsClassProvider optionsProvider) {
+ return ruleClassProvider.getDefaultsPackageContent(optionsProvider);
+ }
+
+ /**
+ * Creates a BuildOptions class for the given options taken from an optionsProvider.
+ */
+ public BuildOptions createBuildOptions(OptionsClassProvider optionsProvider) {
+ return ruleClassProvider.createBuildOptions(optionsProvider);
+ }
+
+ /**
+ * An EventBus exception handler that will report the exception to a remote server, if a
+ * handler is registered.
+ */
+ public static final class RemoteExceptionHandler implements SubscriberExceptionHandler {
+ @Override
+ public void handleException(Throwable exception, SubscriberExceptionContext context) {
+ LoggingUtil.logToRemote(Level.SEVERE, "Failure in EventBus subscriber.", exception);
+ }
+ }
+
+ /**
+ * An EventBus exception handler that will call BugReport.handleCrash exiting
+ * the current thread.
+ */
+ public static final class BugReportingExceptionHandler implements SubscriberExceptionHandler {
+ @Override
+ public void handleException(Throwable exception, SubscriberExceptionContext context) {
+ BugReport.handleCrash(exception);
+ }
+ }
+
+ /**
+ * Main method for the Blaze server startup. Note: This method logs
+ * exceptions to remote servers. Do not add this to a unittest.
+ */
+ public static void main(Iterable<Class<? extends BlazeModule>> moduleClasses, String[] args) {
+ setupUncaughtHandler(args);
+ List<BlazeModule> modules = createModules(moduleClasses);
+ if (args.length >= 1 && args[0].equals("--batch")) {
+ // Run Blaze in batch mode.
+ System.exit(batchMain(modules, args));
+ }
+ LOG.info("Starting Blaze server with args " + Arrays.toString(args));
+ try {
+ // Run Blaze in server mode.
+ System.exit(serverMain(modules, OutErr.SYSTEM_OUT_ERR, args));
+ } catch (RuntimeException | Error e) { // A definite bug...
+ BugReport.printBug(OutErr.SYSTEM_OUT_ERR, e);
+ BugReport.sendBugReport(e, Arrays.asList(args));
+ System.exit(ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode());
+ throw e; // Shouldn't get here.
+ }
+ }
+
+ @VisibleForTesting
+ public static List<BlazeModule> createModules(
+ Iterable<Class<? extends BlazeModule>> moduleClasses) {
+ ImmutableList.Builder<BlazeModule> result = ImmutableList.builder();
+ for (Class<? extends BlazeModule> moduleClass : moduleClasses) {
+ try {
+ BlazeModule module = moduleClass.newInstance();
+ result.add(module);
+ } catch (Throwable e) {
+ throw new IllegalStateException("Cannot instantiate module " + moduleClass.getName(), e);
+ }
+ }
+
+ return result.build();
+ }
+
+ /**
+ * Generates a string form of a request to be written to the logs,
+ * filtering the user environment to remove anything that looks private.
+ * The current filter criteria removes any variable whose name includes
+ * "auth", "pass", or "cookie".
+ *
+ * @param requestStrings
+ * @return the filtered request to write to the log.
+ */
+ @VisibleForTesting
+ public static String getRequestLogString(List<String> requestStrings) {
+ StringBuilder buf = new StringBuilder();
+ buf.append('[');
+ String sep = "";
+ for (String s : requestStrings) {
+ buf.append(sep);
+ if (s.startsWith("--client_env")) {
+ int varStart = "--client_env=".length();
+ int varEnd = s.indexOf('=', varStart);
+ String varName = s.substring(varStart, varEnd);
+ if (suppressFromLog.matcher(varName).matches()) {
+ buf.append("--client_env=");
+ buf.append(varName);
+ buf.append("=__private_value_removed__");
+ } else {
+ buf.append(s);
+ }
+ } else {
+ buf.append(s);
+ }
+ sep = ", ";
+ }
+ buf.append(']');
+ return buf.toString();
+ }
+
+ /**
+ * Command line options split in to two parts: startup options and everything else.
+ */
+ @VisibleForTesting
+ static class CommandLineOptions {
+ private final List<String> startupArgs;
+ private final List<String> otherArgs;
+
+ CommandLineOptions(List<String> startupArgs, List<String> otherArgs) {
+ this.startupArgs = ImmutableList.copyOf(startupArgs);
+ this.otherArgs = ImmutableList.copyOf(otherArgs);
+ }
+
+ public List<String> getStartupArgs() {
+ return startupArgs;
+ }
+
+ public List<String> getOtherArgs() {
+ return otherArgs;
+ }
+ }
+
+ /**
+ * Splits given arguments into two lists - arguments matching options defined in this class
+ * and everything else, while preserving order in each list.
+ */
+ static CommandLineOptions splitStartupOptions(
+ Iterable<BlazeModule> modules, String... args) {
+ List<String> prefixes = new ArrayList<>();
+ List<Field> startupFields = Lists.newArrayList();
+ for (Class<? extends OptionsBase> defaultOptions
+ : BlazeCommandUtils.getStartupOptions(modules)) {
+ startupFields.addAll(ImmutableList.copyOf(defaultOptions.getFields()));
+ }
+
+ for (Field field : startupFields) {
+ if (field.isAnnotationPresent(Option.class)) {
+ prefixes.add("--" + field.getAnnotation(Option.class).name());
+ if (field.getType() == boolean.class || field.getType() == TriState.class) {
+ prefixes.add("--no" + field.getAnnotation(Option.class).name());
+ }
+ }
+ }
+
+ List<String> startupArgs = new ArrayList<>();
+ List<String> otherArgs = Lists.newArrayList(args);
+
+ for (Iterator<String> argi = otherArgs.iterator(); argi.hasNext(); ) {
+ String arg = argi.next();
+ if (!arg.startsWith("--")) {
+ break; // stop at command - all startup options would be specified before it.
+ }
+ for (String prefix : prefixes) {
+ if (arg.startsWith(prefix)) {
+ startupArgs.add(arg);
+ argi.remove();
+ break;
+ }
+ }
+ }
+ return new CommandLineOptions(startupArgs, otherArgs);
+ }
+
+ private static void captureSigint() {
+ final Thread mainThread = Thread.currentThread();
+ final AtomicInteger numInterrupts = new AtomicInteger();
+
+ final Runnable interruptWatcher = new Runnable() {
+ @Override
+ public void run() {
+ int count = 0;
+ // Not an actual infinite loop because it's run in a daemon thread.
+ while (true) {
+ count++;
+ Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS);
+ LOG.warning("Slow interrupt number " + count + " in batch mode");
+ ThreadUtils.warnAboutSlowInterrupt();
+ }
+ }
+ };
+
+ new InterruptSignalHandler() {
+ @Override
+ public void run() {
+ LOG.info("User interrupt");
+ OutErr.SYSTEM_OUT_ERR.printErrLn("Blaze received an interrupt");
+ mainThread.interrupt();
+
+ int curNumInterrupts = numInterrupts.incrementAndGet();
+ if (curNumInterrupts == 1) {
+ Thread interruptWatcherThread = new Thread(interruptWatcher, "interrupt-watcher");
+ interruptWatcherThread.setDaemon(true);
+ interruptWatcherThread.start();
+ } else if (curNumInterrupts == 2) {
+ LOG.warning("Second --batch interrupt: Reverting to JVM SIGINT handler");
+ uninstall();
+ }
+ }
+ };
+ }
+
+ /**
+ * A main method that runs blaze commands in batch mode. The return value indicates the desired
+ * exit status of the program.
+ */
+ private static int batchMain(Iterable<BlazeModule> modules, String[] args) {
+ captureSigint();
+ CommandLineOptions commandLineOptions = splitStartupOptions(modules, args);
+ LOG.info("Running Blaze in batch mode with startup args "
+ + commandLineOptions.getStartupArgs());
+
+ String memoryWarning = validateJvmMemorySettings();
+ if (memoryWarning != null) {
+ OutErr.SYSTEM_OUT_ERR.printErrLn(memoryWarning);
+ }
+
+ BlazeRuntime runtime;
+ try {
+ runtime = newRuntime(modules, parseOptions(modules, commandLineOptions.getStartupArgs()));
+ } catch (OptionsParsingException e) {
+ OutErr.SYSTEM_OUT_ERR.printErr(e.getMessage());
+ return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode();
+ } catch (AbruptExitException e) {
+ OutErr.SYSTEM_OUT_ERR.printErr(e.getMessage());
+ return e.getExitCode().getNumericExitCode();
+ }
+
+ BlazeCommandDispatcher dispatcher =
+ new BlazeCommandDispatcher(runtime, getBuiltinCommandList());
+
+ try {
+ LOG.info(getRequestLogString(commandLineOptions.getOtherArgs()));
+ return dispatcher.exec(commandLineOptions.getOtherArgs(), OutErr.SYSTEM_OUT_ERR,
+ runtime.getClock().currentTimeMillis());
+ } catch (BlazeCommandDispatcher.ShutdownBlazeServerException e) {
+ return e.getExitStatus();
+ } finally {
+ runtime.shutdown();
+ dispatcher.shutdown();
+ }
+ }
+
+ /**
+ * A main method that does not send email. The return value indicates the desired exit status of
+ * the program.
+ */
+ private static int serverMain(Iterable<BlazeModule> modules, OutErr outErr, String[] args) {
+ try {
+ createBlazeRPCServer(modules, Arrays.asList(args)).serve();
+ return ExitCode.SUCCESS.getNumericExitCode();
+ } catch (OptionsParsingException e) {
+ outErr.printErr(e.getMessage());
+ return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode();
+ } catch (IOException e) {
+ outErr.printErr("I/O Error: " + e.getMessage());
+ return ExitCode.BUILD_FAILURE.getNumericExitCode();
+ } catch (AbruptExitException e) {
+ outErr.printErr(e.getMessage());
+ return e.getExitCode().getNumericExitCode();
+ }
+ }
+
+ private static FileSystem fileSystemImplementation() {
+ // The JNI-based UnixFileSystem is faster, but on Windows it is not available.
+ return OS.getCurrent() == OS.WINDOWS ? new JavaIoFileSystem() : new UnixFileSystem();
+ }
+
+ /**
+ * Creates and returns a new Blaze RPCServer. Call {@link RPCServer#serve()} to start the server.
+ */
+ private static RPCServer createBlazeRPCServer(Iterable<BlazeModule> modules, List<String> args)
+ throws IOException, OptionsParsingException, AbruptExitException {
+ OptionsProvider options = parseOptions(modules, args);
+ BlazeServerStartupOptions startupOptions = options.getOptions(BlazeServerStartupOptions.class);
+
+ final BlazeRuntime runtime = newRuntime(modules, options);
+ final BlazeCommandDispatcher dispatcher =
+ new BlazeCommandDispatcher(runtime, getBuiltinCommandList());
+ final String memoryWarning = validateJvmMemorySettings();
+
+ final ServerCommand blazeCommand;
+
+ // Adaptor from RPC mechanism to BlazeCommandDispatcher:
+ blazeCommand = new ServerCommand() {
+ private boolean shutdown = false;
+
+ @Override
+ public int exec(List<String> args, OutErr outErr, long firstContactTime) {
+ LOG.info(getRequestLogString(args));
+ if (memoryWarning != null) {
+ outErr.printErrLn(memoryWarning);
+ }
+
+ try {
+ return dispatcher.exec(args, outErr, firstContactTime);
+ } catch (BlazeCommandDispatcher.ShutdownBlazeServerException e) {
+ if (e.getCause() != null) {
+ StringWriter message = new StringWriter();
+ message.write("Shutting down due to exception:\n");
+ PrintWriter writer = new PrintWriter(message, true);
+ e.printStackTrace(writer);
+ writer.flush();
+ LOG.severe(message.toString());
+ }
+ shutdown = true;
+ runtime.shutdown();
+ dispatcher.shutdown();
+ return e.getExitStatus();
+ }
+ }
+
+ @Override
+ public boolean shutdown() {
+ return shutdown;
+ }
+ };
+
+ RPCServer server = RPCServer.newServerWith(runtime.getClock(), blazeCommand,
+ runtime.getServerDirectory(), runtime.getWorkspace(), startupOptions.maxIdleSeconds);
+ return server;
+ }
+
+ private static Function<String, String> sourceFunctionForMap(final Map<String, String> map) {
+ return new Function<String, String>() {
+ @Override
+ public String apply(String input) {
+ if (!map.containsKey(input)) {
+ return "default";
+ }
+
+ if (map.get(input).isEmpty()) {
+ return "command line";
+ }
+
+ return map.get(input);
+ }
+ };
+ }
+
+ /**
+ * Parses the command line arguments into a {@link OptionsParser} object.
+ *
+ * <p>This function needs to parse the --option_sources option manually so that the real option
+ * parser can set the source for every option correctly. If that cannot be parsed or is missing,
+ * we just report an unknown source for every startup option.
+ */
+ private static OptionsProvider parseOptions(
+ Iterable<BlazeModule> modules, List<String> args) throws OptionsParsingException {
+ Set<Class<? extends OptionsBase>> optionClasses = Sets.newHashSet();
+ optionClasses.addAll(BlazeCommandUtils.getStartupOptions(modules));
+ // First parse the command line so that we get the option_sources argument
+ OptionsParser parser = OptionsParser.newOptionsParser(optionClasses);
+ parser.setAllowResidue(false);
+ parser.parse(OptionPriority.COMMAND_LINE, null, args);
+ Function<? super String, String> sourceFunction =
+ sourceFunctionForMap(parser.getOptions(BlazeServerStartupOptions.class).optionSources);
+
+ // Then parse the command line again, this time with the correct option sources
+ parser = OptionsParser.newOptionsParser(optionClasses);
+ parser.setAllowResidue(false);
+ parser.parseWithSourceFunction(OptionPriority.COMMAND_LINE, sourceFunction, args);
+ return parser;
+ }
+
+ /**
+ * Creates a new blaze runtime, given the install and output base directories.
+ *
+ * <p>Note: This method can and should only be called once per startup, as it also creates the
+ * filesystem object that will be used for the runtime. So it should only ever be called from the
+ * main method of the Blaze program.
+ *
+ * @param options Blaze startup options.
+ *
+ * @return a new BlazeRuntime instance initialized with the given filesystem and directories, and
+ * an error string that, if not null, describes a fatal initialization failure that makes
+ * this runtime unsuitable for real commands
+ */
+ private static BlazeRuntime newRuntime(
+ Iterable<BlazeModule> blazeModules, OptionsProvider options) throws AbruptExitException {
+ for (BlazeModule module : blazeModules) {
+ module.globalInit(options);
+ }
+
+ BlazeServerStartupOptions startupOptions = options.getOptions(BlazeServerStartupOptions.class);
+ PathFragment workspaceDirectory = startupOptions.workspaceDirectory;
+ PathFragment installBase = startupOptions.installBase;
+ PathFragment outputBase = startupOptions.outputBase;
+
+ OsUtils.maybeForceJNI(installBase); // Must be before first use of JNI.
+
+ // From the point of view of the Java program --install_base and --output_base
+ // are mandatory options, despite the comment in their declarations.
+ if (installBase == null || !installBase.isAbsolute()) { // (includes "" default case)
+ throw new IllegalArgumentException(
+ "Bad --install_base option specified: '" + installBase + "'");
+ }
+ if (outputBase != null && !outputBase.isAbsolute()) { // (includes "" default case)
+ throw new IllegalArgumentException(
+ "Bad --output_base option specified: '" + outputBase + "'");
+ }
+
+ PathFragment outputPathFragment = BlazeDirectories.outputPathFromOutputBase(
+ outputBase, workspaceDirectory);
+ FileSystem fs = null;
+ for (BlazeModule module : blazeModules) {
+ FileSystem moduleFs = module.getFileSystem(options, outputPathFragment);
+ if (moduleFs != null) {
+ Preconditions.checkState(fs == null, "more than one module returns a file system");
+ fs = moduleFs;
+ }
+ }
+
+ if (fs == null) {
+ fs = fileSystemImplementation();
+ }
+ Path.setFileSystemForSerialization(fs);
+
+ Path installBasePath = fs.getPath(installBase);
+ Path outputBasePath = fs.getPath(outputBase);
+ Path workspaceDirectoryPath = null;
+ if (!workspaceDirectory.equals(PathFragment.EMPTY_FRAGMENT)) {
+ workspaceDirectoryPath = fs.getPath(workspaceDirectory);
+ }
+
+ BlazeDirectories directories =
+ new BlazeDirectories(installBasePath, outputBasePath, workspaceDirectoryPath);
+
+ Clock clock = BlazeClock.instance();
+
+ BinTools binTools;
+ try {
+ binTools = BinTools.forProduction(directories);
+ } catch (IOException e) {
+ throw new AbruptExitException(
+ "Cannot enumerate embedded binaries: " + e.getMessage(),
+ ExitCode.LOCAL_ENVIRONMENTAL_ERROR);
+ }
+
+ BlazeRuntime.Builder runtimeBuilder = new BlazeRuntime.Builder().setDirectories(directories)
+ .setStartupOptionsProvider(options)
+ .setBinTools(binTools)
+ .setClock(clock)
+ // TODO(bazel-team): Make BugReportingExceptionHandler the default.
+ // See bug "Make exceptions in EventBus subscribers fatal"
+ .setEventBusExceptionHandler(
+ startupOptions.fatalEventBusExceptions || !BlazeVersionInfo.instance().isReleasedBlaze()
+ ? new BlazeRuntime.BugReportingExceptionHandler()
+ : new BlazeRuntime.RemoteExceptionHandler());
+
+ runtimeBuilder.setRunfilesPrefix(new PathFragment(Constants.RUNFILES_PREFIX));
+ for (BlazeModule blazeModule : blazeModules) {
+ runtimeBuilder.addBlazeModule(blazeModule);
+ }
+
+ BlazeRuntime runtime = runtimeBuilder.build();
+ BugReport.setRuntime(runtime);
+ return runtime;
+ }
+
+ /**
+ * Returns null if JVM memory settings are considered safe, and an error string otherwise.
+ */
+ private static String validateJvmMemorySettings() {
+ boolean is64BitVM = "64".equals(System.getProperty("sun.arch.data.model"));
+ if (is64BitVM) {
+ return null;
+ }
+ MemoryMXBean mem = ManagementFactory.getMemoryMXBean();
+ long heapSize = mem.getHeapMemoryUsage().getMax();
+ long nonHeapSize = mem.getNonHeapMemoryUsage().getMax();
+ if (heapSize == -1 || nonHeapSize == -1) {
+ return null;
+ }
+
+ if (heapSize + nonHeapSize > MAX_BLAZE32_RESERVED_MEMORY) {
+ return String.format(
+ "WARNING: JVM reserved %d MB of virtual memory (above threshold of %d MB). "
+ + "This may result in OOMs at runtime. Use lower values of MaxPermSize "
+ + "or switch to blaze64.",
+ (heapSize + nonHeapSize) >> 20, MAX_BLAZE32_RESERVED_MEMORY >> 20);
+ } else if (heapSize < MIN_BLAZE32_HEAP_SIZE) {
+ return String.format(
+ "WARNING: JVM heap size is %d MB. You probably have a custom -Xmx setting in your "
+ + "local Blaze configuration. This may result in OOMs. Removing overrides of -Xmx "
+ + "settings is advised.",
+ heapSize >> 20);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Make sure async threads cannot be orphaned. This method makes sure bugs are reported to
+ * telemetry and the proper exit code is reported.
+ */
+ private static void setupUncaughtHandler(final String[] args) {
+ Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread thread, Throwable throwable) {
+ BugReport.handleCrash(throwable, args);
+ }
+ });
+ }
+
+
+ /**
+ * Returns an immutable list containing new instances of each Blaze command.
+ */
+ @VisibleForTesting
+ public static List<BlazeCommand> getBuiltinCommandList() {
+ return ImmutableList.of(
+ new BuildCommand(),
+ new CanonicalizeCommand(),
+ new CleanCommand(),
+ new HelpCommand(),
+ new SkylarkCommand(),
+ new InfoCommand(),
+ new ProfileCommand(),
+ new QueryCommand(),
+ new RunCommand(),
+ new ShutdownCommand(),
+ new TestCommand(),
+ new VersionCommand());
+ }
+
+ /**
+ * A builder for {@link BlazeRuntime} objects. The only required fields are the {@link
+ * BlazeDirectories}, and the {@link RuleClassProvider} (except for testing). All other fields
+ * have safe default values.
+ *
+ * <p>If a {@link ConfigurationFactory} is set, then the builder ignores the host system flag.
+ * <p>The default behavior of the BlazeRuntime's EventBus is to exit when a subscriber throws
+ * an exception. Please plan appropriately.
+ */
+ public static class Builder {
+
+ private PathFragment runfilesPrefix = PathFragment.EMPTY_FRAGMENT;
+ private BlazeDirectories directories;
+ private Reporter reporter;
+ private ConfigurationFactory configurationFactory;
+ private Clock clock;
+ private OptionsProvider startupOptionsProvider;
+ private final List<BlazeModule> blazeModules = Lists.newArrayList();
+ private SubscriberExceptionHandler eventBusExceptionHandler =
+ new RemoteExceptionHandler();
+ private BinTools binTools;
+ private UUID instanceId;
+
+ public BlazeRuntime build() throws AbruptExitException {
+ Preconditions.checkNotNull(directories);
+ Preconditions.checkNotNull(startupOptionsProvider);
+ Reporter reporter = (this.reporter == null) ? new Reporter() : this.reporter;
+
+ Clock clock = (this.clock == null) ? BlazeClock.instance() : this.clock;
+ UUID instanceId = (this.instanceId == null) ? UUID.randomUUID() : this.instanceId;
+
+ Preconditions.checkNotNull(clock);
+ Map<String, String> clientEnv = new HashMap<>();
+ TimestampGranularityMonitor timestampMonitor = new TimestampGranularityMonitor(clock);
+
+ Preprocessor.Factory.Supplier preprocessorFactorySupplier = null;
+ SkyframeExecutorFactory skyframeExecutorFactory = null;
+ for (BlazeModule module : blazeModules) {
+ module.blazeStartup(startupOptionsProvider,
+ BlazeVersionInfo.instance(), instanceId, directories, clock);
+ Preprocessor.Factory.Supplier modulePreprocessorFactorySupplier =
+ module.getPreprocessorFactorySupplier();
+ if (modulePreprocessorFactorySupplier != null) {
+ Preconditions.checkState(preprocessorFactorySupplier == null,
+ "more than one module defines a preprocessor factory supplier");
+ preprocessorFactorySupplier = modulePreprocessorFactorySupplier;
+ }
+ SkyframeExecutorFactory skyFactory = module.getSkyframeExecutorFactory();
+ if (skyFactory != null) {
+ Preconditions.checkState(skyframeExecutorFactory == null,
+ "At most one skyframe factory supported. But found two: %s and %s", skyFactory,
+ skyframeExecutorFactory);
+ skyframeExecutorFactory = skyFactory;
+ }
+ }
+ if (skyframeExecutorFactory == null) {
+ skyframeExecutorFactory = new SequencedSkyframeExecutorFactory();
+ }
+ if (preprocessorFactorySupplier == null) {
+ preprocessorFactorySupplier = Preprocessor.Factory.Supplier.NullSupplier.INSTANCE;
+ }
+
+ ConfiguredRuleClassProvider.Builder ruleClassBuilder =
+ new ConfiguredRuleClassProvider.Builder();
+ for (BlazeModule module : blazeModules) {
+ module.initializeRuleClasses(ruleClassBuilder);
+ }
+
+ Map<String, String> platformRegexps = null;
+ {
+ ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
+ for (BlazeModule module : blazeModules) {
+ builder.putAll(module.getPlatformSetRegexps());
+ }
+ platformRegexps = builder.build();
+ if (platformRegexps.isEmpty()) {
+ platformRegexps = null; // Use the default.
+ }
+ }
+
+ Set<Path> immutableDirectories = null;
+ {
+ ImmutableSet.Builder<Path> builder = new ImmutableSet.Builder<>();
+ for (BlazeModule module : blazeModules) {
+ builder.addAll(module.getImmutableDirectories());
+ }
+ immutableDirectories = builder.build();
+ }
+
+ Iterable<DiffAwareness.Factory> diffAwarenessFactories = null;
+ {
+ ImmutableList.Builder<DiffAwareness.Factory> builder = new ImmutableList.Builder<>();
+ boolean watchFS = startupOptionsProvider != null
+ && startupOptionsProvider.getOptions(BlazeServerStartupOptions.class).watchFS;
+ for (BlazeModule module : blazeModules) {
+ builder.addAll(module.getDiffAwarenessFactories(watchFS));
+ }
+ diffAwarenessFactories = builder.build();
+ }
+
+ // Merge filters from Blaze modules that allow some action inputs to be missing.
+ Predicate<PathFragment> allowedMissingInputs = null;
+ for (BlazeModule module : blazeModules) {
+ Predicate<PathFragment> modulePredicate = module.getAllowedMissingInputs();
+ if (modulePredicate != null) {
+ Preconditions.checkArgument(allowedMissingInputs == null,
+ "More than one Blaze module allows missing inputs.");
+ allowedMissingInputs = modulePredicate;
+ }
+ }
+ if (allowedMissingInputs == null) {
+ allowedMissingInputs = Predicates.alwaysFalse();
+ }
+
+ ConfiguredRuleClassProvider ruleClassProvider = ruleClassBuilder.build();
+ WorkspaceStatusAction.Factory workspaceStatusActionFactory = null;
+ for (BlazeModule module : blazeModules) {
+ WorkspaceStatusAction.Factory candidate = module.getWorkspaceStatusActionFactory();
+ if (candidate != null) {
+ Preconditions.checkState(workspaceStatusActionFactory == null,
+ "more than one module defines a workspace status action factory");
+ workspaceStatusActionFactory = candidate;
+ }
+ }
+
+ List<PackageFactory.EnvironmentExtension> extensions = new ArrayList<>();
+ for (BlazeModule module : blazeModules) {
+ extensions.add(module.getPackageEnvironmentExtension());
+ }
+
+ // We use an immutable map builder for the nice side effect that it throws if a duplicate key
+ // is inserted.
+ ImmutableMap.Builder<SkyFunctionName, SkyFunction> skyFunctions = ImmutableMap.builder();
+ for (BlazeModule module : blazeModules) {
+ skyFunctions.putAll(module.getSkyFunctions(directories));
+ }
+
+ ImmutableList.Builder<PrecomputedValue.Injected> precomputedValues = ImmutableList.builder();
+ for (BlazeModule module : blazeModules) {
+ precomputedValues.addAll(module.getPrecomputedSkyframeValues());
+ }
+
+ final PackageFactory pkgFactory =
+ new PackageFactory(ruleClassProvider, platformRegexps, extensions);
+ SkyframeExecutor skyframeExecutor = skyframeExecutorFactory.create(reporter, pkgFactory,
+ timestampMonitor, directories, workspaceStatusActionFactory,
+ ruleClassProvider.getBuildInfoFactories(), immutableDirectories, diffAwarenessFactories,
+ allowedMissingInputs, preprocessorFactorySupplier, skyFunctions.build(),
+ precomputedValues.build());
+
+ if (configurationFactory == null) {
+ configurationFactory = new ConfigurationFactory(
+ ruleClassProvider.getConfigurationCollectionFactory(),
+ ruleClassProvider.getConfigurationFragments());
+ }
+
+ ProjectFile.Provider projectFileProvider = null;
+ for (BlazeModule module : blazeModules) {
+ ProjectFile.Provider candidate = module.createProjectFileProvider();
+ if (candidate != null) {
+ Preconditions.checkState(projectFileProvider == null,
+ "more than one module defines a project file provider");
+ projectFileProvider = candidate;
+ }
+ }
+
+ return new BlazeRuntime(directories, reporter, workspaceStatusActionFactory, skyframeExecutor,
+ pkgFactory, ruleClassProvider, configurationFactory,
+ runfilesPrefix == null ? PathFragment.EMPTY_FRAGMENT : runfilesPrefix,
+ clock, startupOptionsProvider, ImmutableList.copyOf(blazeModules),
+ clientEnv, timestampMonitor,
+ eventBusExceptionHandler, binTools, projectFileProvider);
+ }
+
+ public Builder setRunfilesPrefix(PathFragment prefix) {
+ this.runfilesPrefix = prefix;
+ return this;
+ }
+
+ public Builder setBinTools(BinTools binTools) {
+ this.binTools = binTools;
+ return this;
+ }
+
+ public Builder setDirectories(BlazeDirectories directories) {
+ this.directories = directories;
+ return this;
+ }
+
+ /**
+ * Creates and sets a new {@link BlazeDirectories} instance with the given
+ * parameters.
+ */
+ public Builder setDirectories(Path installBase, Path outputBase,
+ Path workspace) {
+ this.directories = new BlazeDirectories(installBase, outputBase, workspace);
+ return this;
+ }
+
+ public Builder setReporter(Reporter reporter) {
+ this.reporter = reporter;
+ return this;
+ }
+
+ public Builder setConfigurationFactory(ConfigurationFactory configurationFactory) {
+ this.configurationFactory = configurationFactory;
+ return this;
+ }
+
+ public Builder setClock(Clock clock) {
+ this.clock = clock;
+ return this;
+ }
+
+ public Builder setStartupOptionsProvider(OptionsProvider startupOptionsProvider) {
+ this.startupOptionsProvider = startupOptionsProvider;
+ return this;
+ }
+
+ public Builder addBlazeModule(BlazeModule blazeModule) {
+ blazeModules.add(blazeModule);
+ return this;
+ }
+
+ public Builder setInstanceId(UUID id) {
+ instanceId = id;
+ return this;
+ }
+
+ @VisibleForTesting
+ public Builder setEventBusExceptionHandler(
+ SubscriberExceptionHandler eventBusExceptionHandler) {
+ this.eventBusExceptionHandler = eventBusExceptionHandler;
+ return this;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java
new file mode 100644
index 0000000..1f9bcea
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java
@@ -0,0 +1,225 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+import java.util.Map;
+
+/**
+ * Options that will be evaluated by the blaze client startup code and passed
+ * to the blaze server upon startup.
+ *
+ * <h4>IMPORTANT</h4> These options and their defaults must be kept in sync with those in the
+ * source of the launcher. The latter define the actual default values; this class exists only to
+ * provide the help message, which displays the default values.
+ *
+ * The same relationship holds between {@link HostJvmStartupOptions} and the launcher.
+ */
+public class BlazeServerStartupOptions extends OptionsBase {
+ /**
+ * Converter for the <code>option_sources</code> option. Takes a string in the form of
+ * "option_name1:source1:option_name2:source2:.." and converts it into an option name to
+ * source map.
+ */
+ public static class OptionSourcesConverter implements Converter<Map<String, String>> {
+ private String unescape(String input) {
+ return input.replace("_C", ":").replace("_U", "_");
+ }
+
+ @Override
+ public Map<String, String> convert(String input) {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ if (input.isEmpty()) {
+ return builder.build();
+ }
+
+ String[] elements = input.split(":");
+ for (int i = 0; i < (elements.length + 1) / 2; i++) {
+ String name = elements[i * 2];
+ String value = "";
+ if (elements.length > i * 2 + 1) {
+ value = elements[i * 2 + 1];
+ }
+ builder.put(unescape(name), unescape(value));
+ }
+ return builder.build();
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "a list of option-source pairs";
+ }
+ }
+
+ /* Passed from the client to the server, specifies the installation
+ * location. The location should be of the form:
+ * $OUTPUT_BASE/_blaze_${USER}/install/${MD5_OF_INSTALL_MANIFEST}.
+ * The server code will only accept a non-empty path; it's the
+ * responsibility of the client to compute a proper default if
+ * necessary.
+ */
+ @Option(name = "install_base",
+ defaultValue = "", // NOTE: purely decorative! See class docstring.
+ category = "hidden",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "This launcher option is intended for use only by tests.")
+ public PathFragment installBase;
+
+ /* Note: The help string in this option applies to the client code; not
+ * the server code. The server code will only accept a non-empty path; it's
+ * the responsibility of the client to compute a proper default if
+ * necessary.
+ */
+ @Option(name = "output_base",
+ defaultValue = "null", // NOTE: purely decorative! See class docstring.
+ category = "server startup",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "If set, specifies the output location to which all build output will be written. "
+ + "Otherwise, the location will be "
+ + "${OUTPUT_ROOT}/_blaze_${USER}/${MD5_OF_WORKSPACE_ROOT}. Note: If you specify a "
+ + "different option from one to the next Blaze invocation for this value, you'll likely "
+ + "start up a new, additional Blaze server. Blaze starts exactly one server per "
+ + "specified output base. Typically there is one output base per workspace--however, "
+ + "with this option you may have multiple output bases per workspace and thereby run "
+ + "multiple builds for the same client on the same machine concurrently. See "
+ + "'blaze help shutdown' on how to shutdown a Blaze server.")
+ public PathFragment outputBase;
+
+ /* Note: This option is only used by the C++ client, never by the Java server.
+ * It is included here to make sure that the option is documented in the help
+ * output, which is auto-generated by Java code.
+ */
+ @Option(name = "output_user_root",
+ defaultValue = "null", // NOTE: purely decorative! See class docstring.
+ category = "server startup",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "The user-specific directory beneath which all build outputs are written; "
+ + "by default, this is a function of $USER, but by specifying a constant, build outputs "
+ + "can be shared between collaborating users.")
+ public PathFragment outputUserRoot;
+
+ @Option(name = "workspace_directory",
+ defaultValue = "",
+ category = "hidden",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "The root of the workspace, that is, the directory that Blaze uses as the root of the "
+ + "build. This flag is only to be set by the blaze client.")
+ public PathFragment workspaceDirectory;
+
+ @Option(name = "max_idle_secs",
+ defaultValue = "" + (3 * 3600), // NOTE: purely decorative! See class docstring.
+ category = "server startup",
+ help = "The number of seconds the build server will wait idling " +
+ "before shutting down. Note: Blaze will ignore this option " +
+ "unless you are starting a new instance. See also 'blaze help " +
+ "shutdown'.")
+ public int maxIdleSeconds;
+
+ @Option(name = "batch",
+ defaultValue = "false", // NOTE: purely decorative! See class docstring.
+ category = "server startup",
+ help = "If set, Blaze will be run in batch mode, instead of " +
+ "the standard client/server. Doing so may provide " +
+ "more predictable semantics with respect to signal handling and job control, " +
+ "Batch mode retains proper queueing semantics within the same output_base. " +
+ "That is, simultaneous invocations will be processed in order, without overlap. " +
+ "If a batch mode Blaze is run on a client with a running server, it first kills " +
+ "the server before processing the command." +
+ "Blaze will run slower in batch mode, compared to client/server mode. " +
+ "Among other things, the build file cache is memory-resident, so it is not " +
+ "preserved between sequential batch invocations. Therefore, using batch mode " +
+ "often makes more sense in cases where performance is less critical, " +
+ "such as continuous builds.")
+ public boolean batch;
+
+ @Option(name = "block_for_lock",
+ defaultValue = "true", // NOTE: purely decorative! See class docstring.
+ category = "server startup",
+ help = "If set, Blaze will exit immediately instead of waiting for other " +
+ "Blaze commands holding the server lock to complete.")
+ public boolean noblock_for_lock;
+
+ @Option(name = "io_nice_level",
+ defaultValue = "-1", // NOTE: purely decorative!
+ category = "server startup",
+ help = "Set a level from 0-7 for best-effort IO scheduling. 0 is highest priority, " +
+ "7 is lowest. The anticipatory scheduler may only honor up to priority 4. " +
+ "Negative values are ignored.")
+ public int ioNiceLevel;
+
+ @Option(name = "batch_cpu_scheduling",
+ defaultValue = "false", // NOTE: purely decorative!
+ category = "server startup",
+ help = "Use 'batch' CPU scheduling for Blaze. This policy is useful for workloads that " +
+ "are non-interactive, but do not want to lower their nice value. " +
+ "See 'man 2 sched_setscheduler'.")
+ public boolean batchCpuScheduling;
+
+ @Option(name = "blazerc",
+ // NOTE: purely decorative!
+ defaultValue = "In the current directory, then in the user's home directory, the file named "
+ + ".$(basename $0)rc (i.e. .bazelrc for Bazel or .blazerc for Blaze)",
+ category = "misc",
+ help = "The location of the .bazelrc/.blazerc file containing default values of "
+ + "Blaze command options. Use /dev/null to disable the search for a "
+ + "blazerc file, e.g. in release builds.")
+ public String blazerc;
+
+ @Option(name = "master_blazerc",
+ defaultValue = "true", // NOTE: purely decorative!
+ category = "misc",
+ help = "If this option is false, the master blazerc/bazelrc next to the binary "
+ + "is not read.")
+ public boolean masterBlazerc;
+
+ @Option(name = "skyframe",
+ defaultValue = "full",
+ category = "undocumented",
+ help = "Unused.")
+ public String unusedSkyframe;
+
+ @Option(name = "fatal_event_bus_exceptions",
+ defaultValue = "false", // NOTE: purely decorative!
+ category = "undocumented",
+ help = "Whether or not to allow EventBus exceptions to be fatal. Experimental.")
+ public boolean fatalEventBusExceptions;
+
+ @Option(name = "option_sources",
+ converter = OptionSourcesConverter.class,
+ defaultValue = "",
+ category = "hidden",
+ help = "")
+ public Map<String, String> optionSources;
+
+ // TODO(bazel-team): In order to make it easier to have local watchers in open source Bazel,
+ // turn this into a non-startup option.
+ @Option(name = "watchfs",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "If true, Blaze tries to use the operating system's file watch service for local "
+ + "changes instead of scanning every file for a change.")
+ public boolean watchFS;
+
+ @Option(name = "use_webstatusserver",
+ defaultValue = "0",
+ category = "server startup",
+ help = "Specifies port to run web status server on (0 to disable, which is default).")
+ public int useWebStatusServer;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java
new file mode 100644
index 0000000..ee1e429
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java
@@ -0,0 +1,141 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.io.OutErr;
+
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Utility methods for sending bug reports.
+ *
+ * <p> Note, code in this class must be extremely robust. There's nothing
+ * worse than a crash-handler that itself crashes!
+ */
+public abstract class BugReport {
+
+ private BugReport() {}
+
+ private static Logger LOG = Logger.getLogger(BugReport.class.getName());
+
+ private static BlazeVersionInfo versionInfo = BlazeVersionInfo.instance();
+
+ private static BlazeRuntime runtime = null;
+
+ public static void setRuntime(BlazeRuntime newRuntime) {
+ Preconditions.checkNotNull(newRuntime);
+ Preconditions.checkState(runtime == null, "runtime already set: %s, %s", runtime, newRuntime);
+ runtime = newRuntime;
+ }
+
+ /**
+ * Logs the unhandled exception with a special prefix signifying that this was a crash.
+ *
+ * @param exception the unhandled exception to display.
+ * @param args additional values to record in the message.
+ * @param values Additional string values to clarify the exception.
+ */
+ public static void sendBugReport(Throwable exception, List<String> args, String... values) {
+ if (!versionInfo.isReleasedBlaze()) {
+ LOG.info("(Not a released binary; not logged.)");
+ return;
+ }
+
+ logException(exception, filterClientEnv(args), values);
+ }
+
+ /**
+ * Print and send a bug report, and exit with the proper Blaze code.
+ */
+ public static void handleCrash(Throwable throwable, String... args) {
+ BugReport.sendBugReport(throwable, Arrays.asList(args));
+ BugReport.printBug(OutErr.SYSTEM_OUT_ERR, throwable);
+ System.err.println("Blaze crash in async thread:");
+ throwable.printStackTrace();
+ int exitCode =
+ (throwable instanceof OutOfMemoryError) ? ExitCode.OOM_ERROR.getNumericExitCode()
+ : ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode();
+ if (runtime != null) {
+ runtime.notifyCommandComplete(exitCode);
+ // We don't call runtime#shutDown() here because all it does is shut down the modules, and who
+ // knows if they can be trusted.
+ }
+ System.exit(exitCode);
+ }
+
+ private static void printThrowableTo(OutErr outErr, Throwable e) {
+ PrintStream err = new PrintStream(outErr.getErrorStream());
+ e.printStackTrace(err);
+ err.flush();
+ LOG.log(Level.SEVERE, "Blaze crashed", e);
+ }
+
+ /**
+ * Print user-helpful information about the bug/crash to the output.
+ *
+ * @param outErr where to write the output
+ * @param e the exception thrown
+ */
+ public static void printBug(OutErr outErr, Throwable e) {
+ if (e instanceof OutOfMemoryError) {
+ outErr.printErr(e.getMessage() + "\n\n" +
+ "Blaze ran out of memory and crashed.\n");
+ } else {
+ printThrowableTo(outErr, e);
+ }
+ }
+
+ /**
+ * Filters {@code args} by removing any item that starts with "--client_env",
+ * then returns this as an immutable list.
+ *
+ * <p>The client's environment variables may contain sensitive data, so we filter it out.
+ */
+ private static List<String> filterClientEnv(Iterable<String> args) {
+ if (args == null) {
+ return null;
+ }
+
+ ImmutableList.Builder<String> filteredArgs = ImmutableList.builder();
+ for (String arg : args) {
+ if (arg != null && !arg.startsWith("--client_env")) {
+ filteredArgs.add(arg);
+ }
+ }
+ return filteredArgs.build();
+ }
+
+ // Log the exception. Because this method is only called in a blaze release,
+ // this will result in a report being sent to a remote logging service.
+ private static void logException(Throwable exception, List<String> args, String... values) {
+ // The preamble is used in the crash watcher, so don't change it
+ // unless you know what you're doing.
+ String preamble = exception instanceof OutOfMemoryError
+ ? "Blaze OOMError: "
+ : "Blaze crashed with args: ";
+
+ LoggingUtil.logToRemote(Level.SEVERE, preamble + Joiner.on(' ').join(args), exception,
+ values);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java b/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java
new file mode 100644
index 0000000..5175a15
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java
@@ -0,0 +1,48 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+/**
+ * Represents how far into the build a given target has gone.
+ * Used primarily for master log status reporting and representation.
+ */
+public enum BuildPhase {
+ PARSING("parsing-failed", false),
+ LOADING("loading-failed", false),
+ ANALYSIS("analysis-failed", false),
+ TEST_FILTERING("test-filtered", true),
+ TARGET_FILTERING("target-filtered", true),
+ NOT_BUILT("not-built", false),
+ NOT_ANALYZED("not-analyzed", false),
+ EXECUTION("build-failed", false),
+ BLAZE_HALTED("blaze-halted", false),
+ COMPLETE("built", true);
+
+ private final String msg;
+ private final boolean success;
+
+ BuildPhase(String msg, boolean success) {
+ this.msg = msg;
+ this.success = success;
+ }
+
+ public String getMessage() {
+ return msg;
+ }
+
+ public boolean getSuccess() {
+ return success;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java b/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java
new file mode 100644
index 0000000..8b072c7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java
@@ -0,0 +1,88 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Joiner;
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.Subscribe;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
+import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.util.BlazeClock;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+/**
+ * Blaze module for the build summary message that reports various stats to the user.
+ */
+public class BuildSummaryStatsModule extends BlazeModule {
+
+ private static final Logger LOG = Logger.getLogger(BuildSummaryStatsModule.class.getName());
+
+ private SimpleCriticalPathComputer criticalPathComputer;
+ private EventBus eventBus;
+ private Reporter reporter;
+
+ @Override
+ public void beforeCommand(BlazeRuntime runtime, Command command) {
+ this.reporter = runtime.getReporter();
+ this.eventBus = runtime.getEventBus();
+ eventBus.register(this);
+ }
+
+ @Subscribe
+ public void executionPhaseStarting(ExecutionStartingEvent event) {
+ criticalPathComputer = new SimpleCriticalPathComputer(BlazeClock.instance());
+ eventBus.register(criticalPathComputer);
+ }
+
+ @Subscribe
+ public void buildComplete(BuildCompleteEvent event) {
+ try {
+ // We might want to make this conditional on a flag; it can sometimes be a bit of a nuisance.
+ List<String> items = new ArrayList<>();
+ items.add(String.format("Elapsed time: %.3fs", event.getResult().getElapsedSeconds()));
+
+ if (criticalPathComputer != null) {
+ Profiler.instance().startTask(ProfilerTask.CRITICAL_PATH, "Critical path");
+ AggregatedCriticalPath<SimpleCriticalPathComponent> criticalPath =
+ criticalPathComputer.aggregate();
+ items.add(criticalPath.toStringSummary());
+ LOG.info(criticalPath.toString());
+ LOG.info("Slowest actions:\n " + Joiner.on("\n ")
+ .join(criticalPathComputer.getSlowestComponents()));
+ // We reverse the critical path because the profiler expect events ordered by the time
+ // when the actions were executed while critical path computation is stored in the reverse
+ // way.
+ for (SimpleCriticalPathComponent stat : criticalPath.components().reverse()) {
+ Profiler.instance().logSimpleTaskDuration(
+ TimeUnit.MILLISECONDS.toNanos(stat.getStartTime()),
+ TimeUnit.MILLISECONDS.toNanos(stat.getActionWallTime()),
+ ProfilerTask.CRITICAL_PATH_COMPONENT, stat.getAction());
+ }
+ Profiler.instance().completeTask(ProfilerTask.CRITICAL_PATH);
+ }
+
+ reporter.handle(Event.info(Joiner.on(", ").join(items)));
+ } finally {
+ criticalPathComputer = null;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/Command.java b/src/main/java/com/google/devtools/build/lib/runtime/Command.java
new file mode 100644
index 0000000..1797cd3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/Command.java
@@ -0,0 +1,108 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation that lets blaze commands specify their options and their help.
+ * The annotations are processed by {@link BlazeCommand}.
+ */
+@Target(ElementType.TYPE)
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Command {
+ /**
+ * The name of the command, as the user would type it.
+ */
+ String name();
+
+ /**
+ * Options processed by the command, indicated by options interfaces.
+ * These interfaces must contain methods annotated with {@link Option}.
+ */
+ Class<? extends OptionsBase>[] options() default {};
+
+ /**
+ * The set of other Blaze commands that this annotation's command "inherits"
+ * options from. These classes must be annotated with {@link Command}.
+ */
+ Class<? extends BlazeCommand>[] inherits() default {};
+
+ /**
+ * A short description, which appears in 'blaze help'.
+ */
+ String shortDescription();
+
+ /**
+ * True if the configuration-specific options should be available for this command.
+ */
+ boolean usesConfigurationOptions() default false;
+
+ /**
+ * True if the command runs a build.
+ */
+ boolean builds() default false;
+
+ /**
+ * True if the command should not be shown in the output of 'blaze help'.
+ */
+ boolean hidden() default false;
+
+ /**
+ * Specifies whether this command allows a residue after the parsed options.
+ * For example, a command might expect a list of targets to build in the
+ * residue.
+ */
+ boolean allowResidue() default false;
+
+ /**
+ * Returns true if this command wants to write binary data to stdout.
+ * Enabling this flag will disable ANSI escape stripping for this command.
+ */
+ boolean binaryStdOut() default false;
+
+ /**
+ * Returns true if this command wants to write binary data to stderr.
+ * Enabling this flag will disable ANSI escape stripping for this command.
+ */
+ boolean binaryStdErr() default false;
+
+ /**
+ * The help message for this command. If the value starts with "resource:",
+ * the remainder is interpreted as the name of a text file resource (in the
+ * .jar file that provides the Command implementation class).
+ */
+ String help();
+
+ /**
+ * Returns true iff this command may only be run from within a Blaze workspace. Broadly, this
+ * should be true for any command that interprets the package-path, since it's potentially
+ * confusing otherwise.
+ */
+ boolean mustRunInWorkspace() default true;
+
+ /**
+ * Returns true iff this command is allowed to run in the output directory,
+ * i.e. $OUTPUT_BASE/_blaze_$USER/$MD5/... . No command should be allowed to run here,
+ * but there are some legacy uses of 'blaze query'.
+ */
+ boolean canRunInOutputDirectory() default false;
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java
new file mode 100644
index 0000000..fb92781
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+/**
+ * This event is fired when the Blaze command is complete
+ * (clean, build, test, etc.).
+ */
+public class CommandCompleteEvent extends CommandEvent {
+
+ private final int exitCode;
+
+ /**
+ * @param exitCode the exit code of the blaze command
+ */
+ public CommandCompleteEvent(int exitCode) {
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * @return the exit code of the blaze command
+ */
+ public int getExitCode() {
+ return exitCode;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java
new file mode 100644
index 0000000..3e59dce
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java
@@ -0,0 +1,68 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.util.BlazeClock;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.util.Date;
+
+/**
+ * Base class for Command events that includes some resource fields.
+ */
+public abstract class CommandEvent {
+
+ private final long eventTimeInNanos;
+ private final long eventTimeInEpochTime;
+ private final long gcTimeInMillis;
+
+ protected CommandEvent() {
+ eventTimeInNanos = BlazeClock.nanoTime();
+ eventTimeInEpochTime = new Date().getTime();
+ gcTimeInMillis = collectGcTimeInMillis();
+ }
+
+ /**
+ * Returns time spent in garbage collection since the start of the JVM process.
+ */
+ private static long collectGcTimeInMillis() {
+ long gcTime = 0;
+ for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) {
+ gcTime += gcBean.getCollectionTime();
+ }
+ return gcTime;
+ }
+
+ /**
+ * Get the time-stamp in ns for the event.
+ */
+ public long getEventTimeInNanos() {
+ return eventTimeInNanos;
+ }
+
+ /**
+ * Get the time-stamp as epoch-time for the event.
+ */
+ public long getEventTimeInEpochTime() {
+ return eventTimeInEpochTime;
+ }
+
+ /**
+ * Get the cumulative GC time for the event.
+ */
+ public long getGCTimeInMillis() {
+ return gcTimeInMillis;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java
new file mode 100644
index 0000000..9a44086
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java
@@ -0,0 +1,38 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.util.ExitCode;
+
+/**
+ * This message is fired right before the Blaze command completes,
+ * and can be used to modify the command's exit code.
+ */
+public class CommandPrecompleteEvent {
+ private final ExitCode exitCode;
+
+ /**
+ * @param exitCode the exit code of the blaze command
+ */
+ public CommandPrecompleteEvent(ExitCode exitCode) {
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * @return the exit code of the blaze command
+ */
+ public ExitCode getExitCode() {
+ return exitCode;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java
new file mode 100644
index 0000000..32834a2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * This event is fired when the Blaze command is started (clean, build, test,
+ * etc.).
+ */
+public class CommandStartEvent extends CommandEvent {
+ private final String commandName;
+ private final UUID commandId;
+ private final Map<String, String> clientEnv;
+ private final Path workingDirectory;
+
+ /**
+ * @param commandName the name of the command
+ */
+ public CommandStartEvent(String commandName, UUID commandId, Map<String, String> clientEnv,
+ Path workingDirectory) {
+ this.commandName = commandName;
+ this.commandId = commandId;
+ this.clientEnv = clientEnv;
+ this.workingDirectory = workingDirectory;
+ }
+
+ public String getCommandName() {
+ return commandName;
+ }
+
+ public UUID getCommandId() {
+ return commandId;
+ }
+
+ public Map<String, String> getClientEnv() {
+ return clientEnv;
+ }
+
+ public Path getWorkingDirectory() {
+ return workingDirectory;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java
new file mode 100644
index 0000000..7054975
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java
@@ -0,0 +1,250 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParsingException;
+
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+
+/**
+ * Options common to all commands.
+ */
+public class CommonCommandOptions extends OptionsBase {
+ /**
+ * A class representing a blazerc option. blazeRc is serial number of the rc
+ * file this option came from, option is the name of the option and value is
+ * its value (or null if not specified).
+ */
+ public static class OptionOverride {
+ final int blazeRc;
+ final String command;
+ final String option;
+
+ public OptionOverride(int blazeRc, String command, String option) {
+ this.blazeRc = blazeRc;
+ this.command = command;
+ this.option = option;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%d:%s=%s", blazeRc, command, option);
+ }
+ }
+
+ /**
+ * Converter for --default_override. The format is:
+ * --default_override=blazerc:command=option.
+ */
+ public static class OptionOverrideConverter implements Converter<OptionOverride> {
+ static final String ERROR_MESSAGE = "option overrides must be in form "
+ + " rcfile:command=option, where rcfile is a nonzero integer";
+
+ public OptionOverrideConverter() {}
+
+ @Override
+ public OptionOverride convert(String input) throws OptionsParsingException {
+ int colonPos = input.indexOf(':');
+ int assignmentPos = input.indexOf('=');
+
+ if (colonPos < 0) {
+ throw new OptionsParsingException(ERROR_MESSAGE);
+ }
+
+ if (assignmentPos <= colonPos + 1) {
+ throw new OptionsParsingException(ERROR_MESSAGE);
+ }
+
+ int blazeRc;
+ try {
+ blazeRc = Integer.valueOf(input.substring(0, colonPos));
+ } catch (NumberFormatException e) {
+ throw new OptionsParsingException(ERROR_MESSAGE);
+ }
+
+ if (blazeRc < 0) {
+ throw new OptionsParsingException(ERROR_MESSAGE);
+ }
+
+ String command = input.substring(colonPos + 1, assignmentPos);
+ String option = input.substring(assignmentPos + 1);
+
+ return new OptionOverride(blazeRc, command, option);
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "blazerc option override";
+ }
+ }
+
+
+ @Option(name = "config",
+ defaultValue = "",
+ category = "misc",
+ allowMultiple = true,
+ help = "Selects additional config sections from the rc files; for every <command>, it "
+ + "also pulls in the options from <command>:<config> if such a section exists. "
+ + "Note that it is currently only possible to provide these options on the "
+ + "command line, not in the rc files. The config sections and flag combinations "
+ + "they are equivalent to are located in the tools/*.blazerc config files.")
+ public List<String> configs;
+
+ @Option(name = "logging",
+ defaultValue = "3", // Level.INFO
+ category = "verbosity",
+ converter = Converters.LogLevelConverter.class,
+ help = "The logging level.")
+ public Level verbosity;
+
+ @Option(name = "client_env",
+ defaultValue = "",
+ category = "hidden",
+ converter = Converters.AssignmentConverter.class,
+ allowMultiple = true,
+ help = "A system-generated parameter which specifies the client's environment")
+ public List<Map.Entry<String, String>> clientEnv;
+
+ @Option(name = "ignore_client_env",
+ defaultValue = "false",
+ category = "hidden",
+ help = "If true, ignore the '--client_env' flag, and use the JVM environment instead")
+ public boolean ignoreClientEnv;
+
+ @Option(name = "client_cwd",
+ defaultValue = "",
+ category = "hidden",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "A system-generated parameter which specifies the client's working directory")
+ public PathFragment clientCwd;
+
+ @Option(name = "announce_rc",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Whether to announce rc options.")
+ public boolean announceRcOptions;
+
+ /**
+ * These are the actual default overrides.
+ * Each value is a pair of (command name, value).
+ *
+ * For example: "--default_override=build=--cpu=piii"
+ */
+ @Option(name = "default_override",
+ defaultValue = "",
+ allowMultiple = true,
+ category = "hidden",
+ converter = OptionOverrideConverter.class,
+ help = "")
+ public List<OptionOverride> optionsOverrides;
+
+ /**
+ * This is the filename that the Blaze client parsed.
+ */
+ @Option(name = "rc_source",
+ defaultValue = "",
+ allowMultiple = true,
+ category = "hidden",
+ help = "")
+ public List<String> rcSource;
+
+ @Option(name = "always_profile_slow_operations",
+ defaultValue = "true",
+ category = "undocumented",
+ help = "Whether profiling slow operations is always turned on")
+ public boolean alwaysProfileSlowOperations;
+
+ @Option(name = "profile",
+ defaultValue = "null",
+ category = "misc",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "If set, profile Blaze and write data to the specified "
+ + "file. Use blaze analyze-profile to analyze the profile.")
+ public PathFragment profilePath;
+
+ @Option(name = "record_full_profiler_data",
+ defaultValue = "false",
+ category = "undocumented",
+ help = "By default, Blaze profiler will record only aggregated data for fast but numerous "
+ + "events (such as statting the file). If this option is enabled, profiler will record "
+ + "each event - resulting in more precise profiling data but LARGE performance "
+ + "hit. Option only has effect if --profile used as well.")
+ public boolean recordFullProfilerData;
+
+ @Option(name = "memory_profile",
+ defaultValue = "null",
+ category = "undocumented",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "If set, write memory usage data to the specified "
+ + "file at phase ends.")
+ public PathFragment memoryProfilePath;
+
+ @Option(name = "gc_watchdog",
+ defaultValue = "false",
+ category = "undocumented",
+ deprecationWarning = "Ignoring: this option is no longer supported",
+ help = "Deprecated.")
+ public boolean gcWatchdog;
+
+ @Option(name = "startup_time",
+ defaultValue = "0",
+ category = "hidden",
+ help = "The time in ms the launcher spends before sending the request to the blaze server.")
+ public long startupTime;
+
+ @Option(name = "extract_data_time",
+ defaultValue = "0",
+ category = "hidden",
+ help = "The time spend on extracting the new blaze version.")
+ public long extractDataTime;
+
+ @Option(name = "command_wait_time",
+ defaultValue = "0",
+ category = "hidden",
+ help = "The time in ms a command had to wait on a busy Blaze server process.")
+ public long waitTime;
+
+ @Option(name = "tool_tag",
+ defaultValue = "",
+ allowMultiple = true,
+ category = "misc",
+ help = "A tool name to attribute this Blaze invocation to.")
+ public List<String> toolTag;
+
+ @Option(name = "restart_reason",
+ defaultValue = "no_restart",
+ category = "hidden",
+ help = "The reason for the server restart.")
+ public String restartReason;
+
+ @Option(name = "binary_path",
+ defaultValue = "",
+ category = "hidden",
+ help = "The absolute path of the blaze binary.")
+ public String binaryPath;
+
+ @Option(name = "experimental_allow_project_files",
+ defaultValue = "false",
+ category = "hidden",
+ help = "Enable processing of +<file> parameters.")
+ public boolean allowProjectFiles;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java b/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java
new file mode 100644
index 0000000..2546492
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java
@@ -0,0 +1,231 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.eventbus.Subscribe;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.actions.ActionCompletionEvent;
+import com.google.devtools.build.lib.actions.ActionMetadata;
+import com.google.devtools.build.lib.actions.ActionMiddlemanEvent;
+import com.google.devtools.build.lib.actions.ActionStartedEvent;
+import com.google.devtools.build.lib.actions.Actions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.CachedActionEvent;
+import com.google.devtools.build.lib.util.Clock;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.PriorityQueue;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.concurrent.ThreadSafe;
+
+/**
+ * Computes the critical path in the action graph based on events published to the event bus.
+ *
+ * <p>After instantiation, this object needs to be registered on the event bus to work.
+ */
+@ThreadSafe
+public abstract class CriticalPathComputer<C extends AbstractCriticalPathComponent<C>,
+ A extends AggregatedCriticalPath<C>> {
+
+ /** Number of top actions to record. */
+ static final int SLOWEST_COMPONENTS_SIZE = 30;
+ // outputArtifactToComponent is accessed from multiple event handlers.
+ protected final ConcurrentMap<Artifact, C> outputArtifactToComponent = Maps.newConcurrentMap();
+
+ /** Maximum critical path found. */
+ private C maxCriticalPath;
+ private final Clock clock;
+
+ /**
+ * The list of slowest individual components, ignoring the time to build dependencies.
+ *
+ * <p>This data is a useful metric when running non highly incremental builds, where multiple
+ * tasks could run un parallel and critical path would only record the longest path.
+ */
+ private final PriorityQueue<C> slowestComponents = new PriorityQueue<>(SLOWEST_COMPONENTS_SIZE,
+ new Comparator<C>() {
+ @Override
+ public int compare(C o1, C o2) {
+ return Long.compare(o1.getActionWallTime(), o2.getActionWallTime());
+ }
+ }
+ );
+
+ private final Object lock = new Object();
+
+ protected CriticalPathComputer(Clock clock) {
+ this.clock = clock;
+ maxCriticalPath = null;
+ }
+
+ /**
+ * Creates a critical path component for an action.
+ * @param action the action for the critical path component
+ * @param startTimeMillis time when the action started to run
+ */
+ protected abstract C createComponent(Action action, long startTimeMillis);
+
+ /**
+ * Return the critical path stats for the current command execution.
+ *
+ * <p>This method allows us to calculate lazily the aggregate statistics of the critical path,
+ * avoiding the memory and cpu penalty for doing it for all the actions executed.
+ */
+ public abstract A aggregate();
+
+ /**
+ * Record an action that has started to run.
+ *
+ * @param event information about the started action
+ */
+ @Subscribe
+ public void actionStarted(ActionStartedEvent event) {
+ Action action = event.getAction();
+ C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart()));
+ for (Artifact output : action.getOutputs()) {
+ C old = outputArtifactToComponent.put(output, component);
+ Preconditions.checkState(old == null, "Duplicate output artifact found. This could happen"
+ + " if a previous event registered the action %s. Artifact: %s", action, output);
+ }
+ }
+
+ /**
+ * Record a middleman action execution. Even if middleman are almost instant, we record them
+ * because they depend on other actions and we need them for constructing the critical path.
+ *
+ * <p>For some rules with incorrect configuration transitions we might get notified several times
+ * for the same middleman. This should only happen if the actions are shared.
+ */
+ @Subscribe
+ public void middlemanAction(ActionMiddlemanEvent event) {
+ Action action = event.getAction();
+ C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart()));
+ boolean duplicate = false;
+ for (Artifact output : action.getOutputs()) {
+ C old = outputArtifactToComponent.putIfAbsent(output, component);
+ if (old != null) {
+ if (!Actions.canBeShared(action, old.getAction())) {
+ throw new IllegalStateException("Duplicate output artifact found for middleman."
+ + "This could happen if a previous event registered the action.\n"
+ + "Old action: " + old.getAction() + "\n\n"
+ + "New action: " + action + "\n\n"
+ + "Artifact: " + output + "\n");
+ }
+ duplicate = true;
+ }
+ }
+ if (!duplicate) {
+ finalizeActionStat(action, component);
+ }
+ }
+
+ /**
+ * Record an action that was not executed because it was in the (disk) cache. This is needed so
+ * that we can calculate correctly the dependencies tree if we have some cached actions in the
+ * middle of the critical path.
+ */
+ @Subscribe
+ public void actionCached(CachedActionEvent event) {
+ Action action = event.getAction();
+ C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart()));
+ for (Artifact output : action.getOutputs()) {
+ outputArtifactToComponent.put(output, component);
+ }
+ finalizeActionStat(action, component);
+ }
+
+ /**
+ * Records the elapsed time stats for the action. For each input artifact, it finds the real
+ * dependent artifacts and records the critical path stats.
+ */
+ @Subscribe
+ public void actionComplete(ActionCompletionEvent event) {
+ ActionMetadata action = event.getActionMetadata();
+ C component = Preconditions.checkNotNull(
+ outputArtifactToComponent.get(action.getPrimaryOutput()));
+ finalizeActionStat(action, component);
+ }
+
+ /** Maximum critical path component found during the build. */
+ protected C getMaxCriticalPath() {
+ synchronized (lock) {
+ return maxCriticalPath;
+ }
+ }
+
+ /**
+ * The list of slowest individual components, ignoring the time to build dependencies.
+ */
+ public ImmutableList<C> getSlowestComponents() {
+ ArrayList<C> list;
+ synchronized (lock) {
+ list = new ArrayList<>(slowestComponents);
+ Collections.sort(list, slowestComponents.comparator());
+ }
+ return ImmutableList.copyOf(list).reverse();
+ }
+
+ private void finalizeActionStat(ActionMetadata action, C component) {
+ component.setFinishTimeMillis(getTime());
+ for (Artifact input : action.getInputs()) {
+ addArtifactDependency(component, input);
+ }
+
+ synchronized (lock) {
+ if (isBiggestCriticalPath(component)) {
+ maxCriticalPath = component;
+ }
+
+ if (slowestComponents.size() == SLOWEST_COMPONENTS_SIZE) {
+ // The new component is faster than any of the slow components, avoid insertion.
+ if (slowestComponents.peek().getActionWallTime() >= component.getActionWallTime()) {
+ return;
+ }
+ // Remove the head element to make space (The fastest component in the queue).
+ slowestComponents.remove();
+ }
+ slowestComponents.add(component);
+ }
+ }
+
+ private long getTime() {
+ return TimeUnit.NANOSECONDS.toMillis(clock.nanoTime());
+ }
+
+ private boolean isBiggestCriticalPath(C newCriticalPath) {
+ synchronized (lock) {
+ return maxCriticalPath == null
+ || maxCriticalPath.getAggregatedWallTime() < newCriticalPath.getAggregatedWallTime();
+ }
+ }
+
+ /**
+ * If "input" is a generated artifact, link its critical path to the one we're building.
+ */
+ private void addArtifactDependency(C actionStats, Artifact input) {
+ C depComponent = outputArtifactToComponent.get(input);
+ if (depComponent != null) {
+ actionStats.addDepInfo(depComponent);
+ }
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java b/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java
new file mode 100644
index 0000000..f4ef8e3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java
@@ -0,0 +1,143 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Preconditions;
+import com.google.devtools.build.lib.events.ExceptionListener;
+import com.google.devtools.build.lib.util.LoggingUtil;
+
+import java.util.logging.Level;
+
+/**
+ * Reports precondition failures from within an event handler.
+ * Necessary because the EventBus silently ignores exceptions thrown from within a handler.
+ * This class logs the exceptions and creates some noise when a precondition check fails.
+ */
+public class EventHandlerPreconditions {
+
+ private final ExceptionListener listener;
+
+ /**
+ * Creates a new precondition helper which outputs errors to the given reporter.
+ */
+ public EventHandlerPreconditions(ExceptionListener listener) {
+ this.listener = listener;
+ }
+
+ /**
+ * Verifies that the given condition (a check on an argument) is true,
+ * throwing an IllegalArgumentException if not.
+ *
+ * @param condition a condition to check for truth.
+ * @throws IllegalArgumentException if the condition is false.
+ */
+ @SuppressWarnings("unused")
+ public void checkArgument(boolean condition) {
+ checkArgument(condition, null);
+ }
+
+ /**
+ * Verifies that the given condition (a check on an argument) is true,
+ * throwing an IllegalArgumentException with the given message if not.
+ *
+ * @param condition a condition to check for truth.
+ * @param message extra information to output if the condition is false.
+ * @throws IllegalArgumentException if the condition is false.
+ */
+ public void checkArgument(boolean condition, String message) {
+ try {
+ Preconditions.checkArgument(condition, message);
+ } catch (IllegalArgumentException iae) {
+ String error = "Event handler argument check failed";
+ LoggingUtil.logToRemote(Level.SEVERE, error, iae);
+ listener.error(null, error, iae);
+ throw iae; // Still terminate the handler.
+ }
+ }
+
+ /**
+ * Verifies that the given condition (a check against the program's current state) is true,
+ * throwing an IllegalStateException if not.
+ *
+ * @param condition a condition to check for truth.
+ * @throws IllegalStateException if the condition is false.
+ */
+ public void checkState(boolean condition) {
+ checkState(condition, null);
+ }
+
+ /**
+ * Verifies that the given condition (a check against the program's current state) is true,
+ * throwing an IllegalStateException with the given message if not.
+ *
+ * @param condition a condition to check for truth.
+ * @param message extra information to output if the condition is false.
+ * @throws IllegalStateException if the condition is false.
+ */
+ public void checkState(boolean condition, String message) {
+ try {
+ Preconditions.checkState(condition, message);
+ } catch (IllegalStateException ise) {
+ String error = "Event handler state check failed";
+ LoggingUtil.logToRemote(Level.SEVERE, error, ise);
+ listener.error(null, error, ise);
+ throw ise; // Still terminate the handler.
+ }
+ }
+
+ /**
+ * Fails with an IllegalStateException when invoked.
+ */
+ public void fail(String message) {
+ String error = "Event handler failed: " + message;
+ IllegalStateException ise = new IllegalStateException(message);
+ LoggingUtil.logToRemote(Level.SEVERE, error, ise);
+ listener.error(null, error, ise);
+ throw ise;
+ }
+
+ /**
+ * Verifies that the given argument is not null, throwing a NullPointerException if it is null.
+ * Returns the original argument or throws.
+ *
+ * @param object an object to test for null.
+ * @return the reference which was checked.
+ * @throws NullPointerException if the object is null.
+ */
+ public <T> T checkNotNull(T object) {
+ return checkNotNull(object, null);
+ }
+
+ /**
+ * Verifies that the given argument is not null, throwing a
+ * NullPointerException with the given message if it is null.
+ * Returns the original argument or throws.
+ *
+ * @param object an object to test for null.
+ * @param message extra information to output if the object is null.
+ * @return the reference which was checked.
+ * @throws NullPointerException if the object is null.
+ */
+ public <T> T checkNotNull(T object, String message) {
+ try {
+ return Preconditions.checkNotNull(object, message);
+ } catch (NullPointerException npe) {
+ String error = "Event handler not-null check failed";
+ LoggingUtil.logToRemote(Level.SEVERE, error, npe);
+ listener.error(null, error, npe);
+ throw npe;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java
new file mode 100644
index 0000000..e55ad2f
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java
@@ -0,0 +1,355 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Splitter;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.AnsiTerminal;
+import com.google.devtools.build.lib.util.io.OutErr;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * An event handler for ANSI terminals which uses control characters to
+ * provide eye-candy, reduce scrolling, and generally improve usability
+ * for users running directly from the shell.
+ *
+ * <p/>
+ * This event handler differs from a normal terminal because it only adds
+ * control characters to stderr, not stdout. All blaze status feedback
+ * is sent to stderr, so adding control characters just to that stream gives
+ * the benefits described above without modifying the normal output stream.
+ * For commands like build that don't generate stdout output this doesn't
+ * matter, but for commands like query and ide_build_info, inserting these
+ * control characters in stdout invalidated their output.
+ *
+ * <p/>
+ * The underlying streams may be either line-bufferred or unbuffered.
+ * Normally each event will write out a sequence of output to a single
+ * stream, and will end with a newline, which ensures a flush.
+ * But care is required when outputting incomplete lines, or when mixing
+ * output between the two different streams (stdout and stderr):
+ * it may be necessary to explicitly flush the output in those cases.
+ * However, we also don't want to flush too often; that can lead to
+ * a choppy UI experience.
+ */
+public class FancyTerminalEventHandler extends BlazeCommandEventHandler {
+ private static Logger LOG = Logger.getLogger(FancyTerminalEventHandler.class.getName());
+ private static final Pattern progressPattern = Pattern.compile(
+ // Match strings that look like they start with progress info:
+ // [42%] Compiling base/base.cc
+ // [1,442 / 23,476] Compiling base/base.cc
+ "^\\[(?:(?:\\d\\d?\\d?%)|(?:[\\d+,]+ / [\\d,]+))\\] ");
+ private static final Splitter LINEBREAK_SPLITTER = Splitter.on('\n');
+
+ private final AnsiTerminal terminal;
+
+ private final boolean useColor;
+ private final boolean useCursorControls;
+ private final boolean progressInTermTitle;
+ public final int terminalWidth;
+
+ private boolean terminalClosed = false;
+ private boolean previousLineErasable = false;
+ private int numLinesPreviousErasable = 0;
+
+ public FancyTerminalEventHandler(OutErr outErr, BlazeCommandEventHandler.Options options) {
+ super(outErr, options);
+ this.terminal = new AnsiTerminal(outErr.getErrorStream());
+ this.terminalWidth = (options.terminalColumns > 0 ? options.terminalColumns : 80);
+ useColor = options.useColor();
+ useCursorControls = options.useCursorControl();
+ progressInTermTitle = options.progressInTermTitle;
+ }
+
+ @Override
+ public void handle(Event event) {
+ if (terminalClosed) {
+ return;
+ }
+ if (!eventMask.contains(event.getKind())) {
+ return;
+ }
+
+ try {
+ boolean previousLineErased = false;
+ if (previousLineErasable) {
+ previousLineErased = maybeOverwritePreviousMessage();
+ }
+ switch (event.getKind()) {
+ case PROGRESS:
+ case START:
+ {
+ String message = event.getMessage();
+ Pair<String,String> progressPair = matchProgress(message);
+ if (progressPair != null) {
+ progress(progressPair.getFirst(), progressPair.getSecond());
+ } else {
+ progress("INFO: ", message);
+ }
+ break;
+ }
+ case FINISH:
+ {
+ String message = event.getMessage();
+ Pair<String,String> progressPair = matchProgress(message);
+ if (progressPair != null) {
+ String percentage = progressPair.getFirst();
+ String rest = progressPair.getSecond();
+ progress(percentage, rest + " DONE");
+ } else {
+ progress("INFO: ", message + " DONE");
+ }
+ break;
+ }
+ case PASS:
+ progress("PASS: ", event.getMessage());
+ break;
+ case INFO:
+ info(event);
+ break;
+ case ERROR:
+ case FAIL:
+ case TIMEOUT:
+ // For errors, scroll the message, so it appears above the status
+ // line, and highlight the word "ERROR" or "FAIL" in boldface red.
+ errorOrFail(event);
+ break;
+ case WARNING:
+ // For warnings, highlight the word "Warning" in boldface magenta,
+ // and scroll it.
+ warning(event);
+ break;
+ case SUBCOMMAND:
+ subcmd(event);
+ break;
+ case STDOUT:
+ if (previousLineErased) {
+ terminal.flush();
+ }
+ previousLineErasable = false;
+ super.handle(event);
+ // We don't need to flush stdout here, because
+ // super.handle(event) will take care of that.
+ break;
+ case STDERR:
+ putOutput(event);
+ break;
+ default:
+ // Ignore all other event types.
+ break;
+ }
+ } catch (IOException e) {
+ // The terminal shouldn't have IO errors, unless the shell is killed, which
+ // should also kill the blaze client. So this isn't something that should
+ // occur here; it will show up in the client/server interface as a broken
+ // pipe.
+ LOG.warning("Terminal was closed during build: " + e);
+ terminalClosed = true;
+ }
+ }
+
+ /**
+ * Displays a progress message that may be erased by subsequent messages.
+ *
+ * @param prefix a short string such as "[99%] " or "INFO: ", which will be highlighted
+ * @param rest the remainder of the message; may be multiple lines
+ */
+ private void progress(String prefix, String rest) throws IOException {
+ previousLineErasable = true;
+
+ if (progressInTermTitle) {
+ int newlinePos = rest.indexOf('\n');
+ if (newlinePos == -1) {
+ terminal.setTitle(prefix + rest);
+ } else {
+ terminal.setTitle(prefix + rest.substring(0, newlinePos));
+ }
+ }
+
+ if (useColor) {
+ terminal.textGreen();
+ }
+ int prefixWidth = prefix.length();
+ terminal.writeString(prefix);
+ terminal.resetTerminal();
+ if (showTimestamp) {
+ String timestamp = timestamp();
+ prefixWidth += timestamp.length();
+ terminal.writeString(timestamp);
+ }
+ int numLines = 0;
+ Iterator<String> lines = LINEBREAK_SPLITTER.split(rest).iterator();
+ String firstLine = lines.next();
+ terminal.writeString(firstLine);
+ // Subtract one, because when the line length is the same as the terminal
+ // width, the terminal doesn't line-advance, so we don't want to erase
+ // two lines.
+ numLines += (prefixWidth + firstLine.length() - 1) / terminalWidth + 1;
+ crlf();
+ while (lines.hasNext()) {
+ String line = lines.next();
+ terminal.writeString(line);
+ crlf();
+ numLines += (line.length() - 1) / terminalWidth + 1;
+ }
+ numLinesPreviousErasable = numLines;
+ }
+
+ /**
+ * Try to match a message against the "progress message" pattern. If it
+ * matches, return the progress percentage, and the rest of the message.
+ * @param message the message to match
+ * @return a pair containing the progress percentage, and the rest of the
+ * progress message, or null if the message isn't a progress message.
+ */
+ private Pair<String,String> matchProgress(String message) {
+ Matcher m = progressPattern.matcher(message);
+ if (m.find()) {
+ return Pair.of(message.substring(0, m.end()), message.substring(m.end()));
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Send the terminal controls that will put the cursor on the beginning
+ * of the same line if cursor control is on, or the next line if not.
+ * @returns True if it did any output; if so, caller is responsible for
+ * flushing the terminal if needed.
+ */
+ private boolean maybeOverwritePreviousMessage() throws IOException {
+ if (useCursorControls && numLinesPreviousErasable != 0) {
+ for (int i = 0; i < numLinesPreviousErasable; i++) {
+ terminal.cr();
+ terminal.cursorUp(1);
+ terminal.clearLine();
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private void errorOrFail(Event event) throws IOException {
+ previousLineErasable = false;
+ if (useColor) {
+ terminal.textRed();
+ terminal.textBold();
+ }
+ terminal.writeString(event.getKind().toString() + ": ");
+ if (useColor) {
+ terminal.resetTerminal();
+ }
+ writeTimestampAndLocation(event);
+ terminal.writeString(event.getMessage());
+ terminal.writeString(".");
+ crlf();
+ }
+
+ private void warning(Event warning) throws IOException {
+ previousLineErasable = false;
+ if (useColor) {
+ terminal.textMagenta();
+ }
+ terminal.writeString("WARNING: ");
+ terminal.resetTerminal();
+ writeTimestampAndLocation(warning);
+ terminal.writeString(warning.getMessage());
+ terminal.writeString(".");
+ crlf();
+ }
+
+ private void info(Event event) throws IOException {
+ previousLineErasable = false;
+ if (useColor) {
+ terminal.textGreen();
+ }
+ terminal.writeString(event.getKind().toString() + ": ");
+ terminal.resetTerminal();
+ writeTimestampAndLocation(event);
+ terminal.writeString(event.getMessage());
+ // No period; info messages often end in '...'.
+ crlf();
+ }
+
+ private void subcmd(Event subcmd) throws IOException {
+ previousLineErasable = false;
+ if (useColor) {
+ terminal.textBlue();
+ }
+ terminal.writeString(">>>>> ");
+ terminal.resetTerminal();
+ writeTimestampAndLocation(subcmd);
+ terminal.writeString(subcmd.getMessage());
+ crlf();
+ }
+
+ /* Handle STDERR events. */
+ private void putOutput(Event event) throws IOException {
+ previousLineErasable = false;
+ terminal.writeBytes(event.getMessageBytes());
+/*
+ * The following code doesn't work because buildtool.TerminalTestNotifier
+ * writes ANSI-formatted text via this mechanism, one character at a time,
+ * and if we try to insert additional ANSI sequences in between the characters
+ * of another ANSI escape sequence, we screw things up. (?)
+ * TODO(bazel-team): (2009) fix this. TerminalTestNotifier should go via the Reporter
+ * rather than via an AnsiTerminalWriter.
+ */
+// terminal.resetTerminal();
+// writeTimestampAndLocation(event);
+// if (useColor) {
+// terminal.textNormal();
+// }
+// terminal.writeBytes(event.getMessageBytes());
+// terminal.resetTerminal();
+ }
+
+ /**
+ * Add a carriage return, shifting to the next line on the terminal, while
+ * guaranteeing that the terminal control codes don't cause any strange
+ * effects. Without the CR before the "\n", the "\n" can cause a line-break
+ * moving text to the next line, where the new message will be generated.
+ * Emitting a "CR" before means that the actual terminal controls generated
+ * here are CR+CR+LF; the double-CR resets the terminal line state, which
+ * prevents the potentially ugly formatting issue.
+ */
+ private void crlf() throws IOException {
+ terminal.cr();
+ terminal.writeString("\n");
+ }
+
+ private void writeTimestampAndLocation(Event event) throws IOException {
+ if (showTimestamp) {
+ terminal.writeString(timestamp());
+ }
+ if (event.getLocation() != null) {
+ terminal.writeString(event.getLocation() + ": ");
+ }
+ }
+
+ public void resetTerminal() {
+ try {
+ terminal.resetTerminal();
+ } catch (IOException e) {
+ LOG.warning("IO Error writing to user terminal: " + e);
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java b/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java
new file mode 100644
index 0000000..48e366d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java
@@ -0,0 +1,85 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Record GC stats for a build.
+ */
+public class GCStatsRecorder {
+
+ private final Iterable<GarbageCollectorMXBean> mxBeans;
+ private final ImmutableMap<String, GCStat> initialData;
+
+ public GCStatsRecorder(Iterable<GarbageCollectorMXBean> mxBeans) {
+ this.mxBeans = mxBeans;
+ ImmutableMap.Builder<String, GCStat> initialData = ImmutableMap.builder();
+ for (GarbageCollectorMXBean mxBean : mxBeans) {
+ String name = mxBean.getName();
+ initialData.put(name, new GCStat(name, mxBean.getCollectionCount(),
+ mxBean.getCollectionTime()));
+ }
+ this.initialData = initialData.build();
+ }
+
+ public Iterable<GCStat> getCurrentGcStats() {
+ List<GCStat> stats = new ArrayList<>();
+ for (GarbageCollectorMXBean mxBean : mxBeans) {
+ String name = mxBean.getName();
+ GCStat initStat = Preconditions.checkNotNull(initialData.get(name));
+ stats.add(new GCStat(name,
+ mxBean.getCollectionCount() - initStat.getNumCollections(),
+ mxBean.getCollectionTime() - initStat.getTotalTimeInMs()));
+ }
+ return stats;
+ }
+
+ /** Represents the garbage collections statistics for one collector (For example CMS). */
+ public static class GCStat {
+
+ private final String name;
+ private final long numCollections;
+ private final long totalTimeInMs;
+
+ public GCStat(String name, long numCollections, long totalTimeInMs) {
+ this.name = name;
+ this.numCollections = numCollections;
+ this.totalTimeInMs = totalTimeInMs;
+ }
+
+ /** Name of the Collector. For example CMS. */
+ public String getName() { return name; }
+
+ /** Number of invocations for a build. */
+ public long getNumCollections() { return numCollections; }
+
+ /**
+ * Total time spend in GC for the collector. Note that the time does need to be exclusive (aka a
+ * stop-the-world GC).
+ */
+ public long getTotalTimeInMs() { return totalTimeInMs; }
+
+ @Override
+ public String toString() {
+ return "GC time for '" + name + "' collector: " + numCollections
+ + " collections using " + totalTimeInMs + "ms";
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java
new file mode 100644
index 0000000..622d112
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java
@@ -0,0 +1,51 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.common.options.OptionsProvider;
+
+/**
+ * An event in which the command line options
+ * are discovered.
+ */
+public class GotOptionsEvent {
+
+ private final OptionsProvider startupOptions;
+ private final OptionsProvider options;
+
+ /**
+ * Construct the options event.
+ *
+ * @param startupOptions the parsed startup options
+ * @param options the parsed options
+ */
+ public GotOptionsEvent(OptionsProvider startupOptions, OptionsProvider options) {
+ this.startupOptions = startupOptions;
+ this.options = options;
+ }
+
+ /**
+ * @return the parsed startup options
+ */
+ public OptionsProvider getStartupOptions() {
+ return startupOptions;
+ }
+
+ /**
+ * @return the parsed options.
+ */
+ public OptionsProvider getOptions() {
+ return options;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java
new file mode 100644
index 0000000..305c048
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+/**
+ * Options that will be evaluated by the blaze client startup code only.
+ *
+ * The only reason we have this interface is that we'd like to print a nice
+ * help page for the client startup options. These options do not affect the
+ * server's behavior in any way.
+ */
+public class HostJvmStartupOptions extends OptionsBase {
+
+ @Option(name = "host_jvm_args",
+ defaultValue = "", // NOTE: purely decorative! See BlazeServerStartupOptions.
+ category = "host jvm startup",
+ help = "Flags to pass to the JVM executing Blaze. Note: Blaze " +
+ "will ignore this option unless you are starting a new " +
+ "instance. See also 'blaze help shutdown'.")
+ public String hostJvmArgs;
+
+ @Option(name = "host_jvm_profile",
+ defaultValue = "", // NOTE: purely decorative! See BlazeServerStartupOptions.
+ category = "host jvm startup",
+ help = "Run the JVM executing Blaze in the given profiler. " +
+ "Blaze will search for hardcoded paths based on the " +
+ "profiler. Note: Blaze will ignore this option unless you " +
+ "are starting a new instance. See also 'blaze help shutdown'.")
+ public String hostJvmProfile;
+
+ @Option(name = "host_jvm_debug",
+ defaultValue = "false", // NOTE: purely decorative! See BlazeServerStartupOptions.
+ category = "host jvm startup",
+ help = "Run the JVM executing Blaze so that it listens for a " +
+ "connection from a JDWP-compliant debugger. Note: Blaze " +
+ "will ignore this option unless you are starting a new " +
+ "instance. See also 'blaze help shutdown'.")
+ public boolean hostJvmDebug;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java b/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java
new file mode 100644
index 0000000..56747d8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.List;
+
+/**
+ * A file that describes a project - for large source trees that are worked on by multiple
+ * independent teams, it is useful to have a larger unit than a package which combines a set of
+ * target patterns and a set of corresponding options.
+ */
+public interface ProjectFile {
+
+ /**
+ * A provider for a project file - we generally expect the provider to cache parsed files
+ * internally and return a cached version if it can ascertain that that is still correct.
+ *
+ * <p>Note in particular that packages may be moved between different package path entries, which
+ * should lead to cache invalidation.
+ */
+ public interface Provider {
+ /**
+ * Returns an (optionally cached) project file instance. If there is no such file, or if the
+ * file cannot be parsed, then it throws an exception.
+ */
+ ProjectFile getProjectFile(List<Path> packagePath, PathFragment path)
+ throws AbruptExitException;
+ }
+
+ /**
+ * A string name of the project file that is reported to the user. It should be in such a format
+ * that passing it back in on the command line works.
+ */
+ String getName();
+
+ /**
+ * A list of strings that are parsed into the options for the command.
+ *
+ * @param command An action from the command line, e.g. "build" or "test".
+ * @throws UnsupportedOperationException if an unknown command is passed.
+ */
+ List<String> getCommandLineFor(String command);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java
new file mode 100644
index 0000000..5e90f2e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Clock;
+
+/**
+ * An event handler that rate limits events.
+ */
+public class RateLimitingEventHandler implements EventHandler {
+
+ private final EventHandler outputHandler;
+ private final double intervalMillis;
+ private final Clock clock;
+ private long lastEventMillis = -1;
+
+ /**
+ * Creates a new Event handler that rate limits the events of type PROGRESS
+ * to one per event "rateLimitation" seconds. Events that arrive too quickly are dropped;
+ * all others are are forwarded to the handler "delegateTo".
+ *
+ * @param delegateTo The event handler that ultimately handles the events
+ * @param rateLimitation The minimum number of seconds between events that will be forwarded
+ * to the delegateTo-handler.
+ * If less than zero (or NaN), all events will be forwarded.
+ */
+ public static EventHandler create(EventHandler delegateTo, double rateLimitation) {
+ if (rateLimitation < 0.0 || Double.isNaN(rateLimitation)) {
+ return delegateTo;
+ }
+ return new RateLimitingEventHandler(delegateTo, rateLimitation);
+ }
+
+ private RateLimitingEventHandler(EventHandler delegateTo, double rateLimitation) {
+ clock = BlazeClock.instance();
+ outputHandler = delegateTo;
+ this.intervalMillis = rateLimitation * 1000;
+ }
+
+ @Override
+ public void handle(Event event) {
+ switch (event.getKind()) {
+ case PROGRESS:
+ case START:
+ case FINISH:
+ long currentTime = clock.currentTimeMillis();
+ if (lastEventMillis + intervalMillis <= currentTime) {
+ lastEventMillis = currentTime;
+ outputHandler.handle(event);
+ }
+ break;
+ default:
+ outputHandler.handle(event);
+ break;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java
new file mode 100644
index 0000000..b8d5d45
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java
@@ -0,0 +1,26 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.actions.Action;
+
+/**
+ * This class records the critical path for the graph of actions executed.
+ */
+public class SimpleCriticalPathComponent
+ extends AbstractCriticalPathComponent<SimpleCriticalPathComponent> {
+
+ public SimpleCriticalPathComponent(Action action, long startTime) { super(action, startTime); }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java
new file mode 100644
index 0000000..65a9c95
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java
@@ -0,0 +1,58 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.util.Clock;
+
+/**
+ * Computes the critical path during a build.
+ */
+public class SimpleCriticalPathComputer
+ extends CriticalPathComputer<SimpleCriticalPathComponent,
+ AggregatedCriticalPath<SimpleCriticalPathComponent>> {
+
+ public SimpleCriticalPathComputer(Clock clock) {
+ super(clock);
+ }
+
+ @Override
+ public SimpleCriticalPathComponent createComponent(Action action, long startTimeMillis) {
+ return new SimpleCriticalPathComponent(action, startTimeMillis);
+ }
+
+ /**
+ * Return the critical path stats for the current command execution.
+ *
+ * <p>This method allow us to calculate lazily the aggregate statistics of the critical path,
+ * avoiding the memory and cpu penalty for doing it for all the actions executed.
+ */
+ @Override
+ public AggregatedCriticalPath<SimpleCriticalPathComponent> aggregate() {
+ ImmutableList.Builder<SimpleCriticalPathComponent> components = ImmutableList.builder();
+ SimpleCriticalPathComponent maxCriticalPath = getMaxCriticalPath();
+ if (maxCriticalPath == null) {
+ return new AggregatedCriticalPath<>(0, components.build());
+ }
+ SimpleCriticalPathComponent child = maxCriticalPath;
+ while (child != null) {
+ components.add(child);
+ child = child.getChild();
+ }
+ return new AggregatedCriticalPath<>(maxCriticalPath.getAggregatedWallTime(),
+ components.build());
+ }
+}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java b/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java
new file mode 100644
index 0000000..0134f55
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java
@@ -0,0 +1,220 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.rules.test.TestLogHelper;
+import com.google.devtools.build.lib.rules.test.TestResult;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestSummaryFormat;
+import com.google.devtools.build.lib.util.StringUtil;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Prints the test results to a terminal.
+ */
+public class TerminalTestResultNotifier implements TestResultNotifier {
+ private static class TestResultStats {
+ int numberOfTargets;
+ int passCount;
+ int failedToBuildCount;
+ int failedCount;
+ int failedRemotelyCount;
+ int failedLocallyCount;
+ int noStatusCount;
+ int numberOfExecutedTargets;
+ boolean wasUnreportedWrongSize;
+ }
+
+ /**
+ * Flags specific to test summary reporting.
+ */
+ public static class TestSummaryOptions extends OptionsBase {
+ @Option(name = "verbose_test_summary",
+ defaultValue = "true",
+ category = "verbosity",
+ help = "If true, print additional information (timing, number of failed runs, etc) in the"
+ + " test summary.")
+ public boolean verboseSummary;
+
+ @Option(name = "test_verbose_timeout_warnings",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "If true, print additional warnings when the actual test execution time does not " +
+ "match the timeout defined by the test (whether implied or explicit).")
+ public boolean testVerboseTimeoutWarnings;
+ }
+
+ private final AnsiTerminalPrinter printer;
+ private final OptionsProvider options;
+ private final TestSummaryOptions summaryOptions;
+
+ /**
+ * @param printer The terminal to print to
+ */
+ public TerminalTestResultNotifier(AnsiTerminalPrinter printer, OptionsProvider options) {
+ this.printer = printer;
+ this.options = options;
+ this.summaryOptions = options.getOptions(TestSummaryOptions.class);
+ }
+
+ /**
+ * Prints a test result summary that contains only failed tests.
+ */
+ private void printDetailedTestResultSummary(Set<TestSummary> summaries) {
+ for (TestSummary entry : summaries) {
+ if (entry.getStatus() != BlazeTestStatus.PASSED) {
+ TestSummaryPrinter.print(entry, printer, summaryOptions.verboseSummary, true);
+ }
+ }
+ }
+
+ /**
+ * Prints a full test result summary.
+ */
+ private void printShortSummary(Set<TestSummary> summaries, boolean showPassingTests) {
+ for (TestSummary entry : summaries) {
+ if (entry.getStatus() != BlazeTestStatus.PASSED || showPassingTests) {
+ TestSummaryPrinter.print(entry, printer, summaryOptions.verboseSummary, false);
+ }
+ }
+ }
+
+ /**
+ * Returns true iff the --check_tests_up_to_date option is enabled.
+ */
+ private boolean optionCheckTestsUpToDate() {
+ return options.getOptions(ExecutionOptions.class).testCheckUpToDate;
+ }
+
+
+ /**
+ * Prints a test summary information for all tests to the terminal.
+ *
+ * @param summaries Summary of all targets that were ran
+ * @param numberOfExecutedTargets the number of targets that were actually ran
+ */
+ @Override
+ public void notify(Set<TestSummary> summaries, int numberOfExecutedTargets) {
+ TestResultStats stats = new TestResultStats();
+ stats.numberOfTargets = summaries.size();
+ stats.numberOfExecutedTargets = numberOfExecutedTargets;
+
+ TestOutputFormat testOutput = options.getOptions(ExecutionOptions.class).testOutput;
+
+ for (TestSummary summary : summaries) {
+ if (summary.isLocalActionCached()
+ && TestLogHelper.shouldOutputTestLog(testOutput,
+ TestResult.isBlazeTestStatusPassed(summary.getStatus()))) {
+ TestSummaryPrinter.printCachedOutput(summary, testOutput, printer);
+ }
+ }
+
+ for (TestSummary summary : summaries) {
+ if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) {
+ stats.passCount++;
+ } else if (summary.getStatus() == BlazeTestStatus.FAILED_TO_BUILD) {
+ stats.failedToBuildCount++;
+ } else if (summary.ranRemotely()) {
+ stats.failedRemotelyCount++;
+ } else {
+ stats.failedLocallyCount++;
+ }
+
+ if (summary.getStatus() == BlazeTestStatus.NO_STATUS) {
+ stats.noStatusCount++;
+ }
+
+ if (summary.wasUnreportedWrongSize()) {
+ stats.wasUnreportedWrongSize = true;
+ }
+ }
+
+ stats.failedCount = summaries.size() - stats.passCount;
+
+ TestSummaryFormat testSummaryFormat = options.getOptions(ExecutionOptions.class).testSummary;
+ switch (testSummaryFormat) {
+ case DETAILED:
+ printDetailedTestResultSummary(summaries);
+ break;
+
+ case SHORT:
+ printShortSummary(summaries, /*printSuccess=*/true);
+ break;
+
+ case TERSE:
+ printShortSummary(summaries, /*printSuccess=*/false);
+ break;
+
+ case NONE:
+ break;
+ }
+
+ printStats(stats);
+ }
+
+ private void addToErrorList(List<String> list, String failureDescription, int count) {
+ if (count > 0) {
+ list.add(String.format("%s%d %s %s%s",
+ AnsiTerminalPrinter.Mode.ERROR,
+ count,
+ count == 1 ? "fails" : "fail",
+ failureDescription,
+ AnsiTerminalPrinter.Mode.DEFAULT));
+ }
+ }
+
+ private void printStats(TestResultStats stats) {
+ if (!optionCheckTestsUpToDate()) {
+ List<String> results = new ArrayList<>();
+ if (stats.passCount == 1) {
+ results.add(stats.passCount + " test passes");
+ } else if (stats.passCount > 0) {
+ results.add(stats.passCount + " tests pass");
+ }
+ addToErrorList(results, "to build", stats.failedToBuildCount);
+ addToErrorList(results, "locally", stats.failedLocallyCount);
+ addToErrorList(results, "remotely", stats.failedRemotelyCount);
+ printer.print(String.format("\nExecuted %d out of %d tests: %s.\n",
+ stats.numberOfExecutedTargets,
+ stats.numberOfTargets,
+ StringUtil.joinEnglishList(results, "and")));
+ } else {
+ int failingUpToDateCount = stats.failedCount - stats.noStatusCount;
+ printer.print(String.format(
+ "\nFinished with %d passing and %s%d failing%s tests up to date, %s%d out of date.%s\n",
+ stats.passCount,
+ failingUpToDateCount > 0 ? AnsiTerminalPrinter.Mode.ERROR : "",
+ failingUpToDateCount,
+ AnsiTerminalPrinter.Mode.DEFAULT,
+ stats.noStatusCount > 0 ? AnsiTerminalPrinter.Mode.ERROR : "",
+ stats.noStatusCount,
+ AnsiTerminalPrinter.Mode.DEFAULT));
+ }
+
+ if (stats.wasUnreportedWrongSize) {
+ printer.print("There were tests whose specified size is too big. Use the "
+ + "--test_verbose_timeout_warnings command line option to see which "
+ + "ones these are.\n");
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java b/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java
new file mode 100644
index 0000000..ed9120b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java
@@ -0,0 +1,349 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.packages.TestSize;
+import com.google.devtools.build.lib.packages.TestTimeout;
+import com.google.devtools.build.lib.rules.test.TestProvider;
+import com.google.devtools.build.lib.rules.test.TestResult;
+import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions;
+import com.google.devtools.build.lib.syntax.Label;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Prints results to the terminal, showing the results of each test target.
+ */
+@ThreadCompatible
+public class TestResultAnalyzer {
+ private final Path execRoot;
+ private final TestSummaryOptions summaryOptions;
+ private final ExecutionOptions executionOptions;
+ private final EventBus eventBus;
+
+ /**
+ * @param summaryOptions Parsed test summarization options.
+ * @param executionOptions Parsed build/test execution options.
+ * @param eventBus For reporting failed to build and cached tests.
+ */
+ public TestResultAnalyzer(Path execRoot,
+ TestSummaryOptions summaryOptions,
+ ExecutionOptions executionOptions,
+ EventBus eventBus) {
+ this.execRoot = execRoot;
+ this.summaryOptions = summaryOptions;
+ this.executionOptions = executionOptions;
+ this.eventBus = eventBus;
+ }
+
+ /**
+ * Prints out the results of the given tests, and returns true if they all passed.
+ * Posts any targets which weren't already completed by the listener to the EventBus.
+ * Reports all targets on the console via the given notifier.
+ * Run at the end of the build, run only once.
+ *
+ * @param testTargets The list of targets being run
+ * @param listener An aggregating listener with intermediate results
+ * @param notifier A console notifier to echo results to.
+ * @return true if all the tests passed, else false
+ */
+ public boolean differentialAnalyzeAndReport(
+ Collection<ConfiguredTarget> testTargets,
+ AggregatingTestListener listener,
+ TestResultNotifier notifier) {
+
+ Preconditions.checkNotNull(testTargets);
+ Preconditions.checkNotNull(listener);
+ Preconditions.checkNotNull(notifier);
+
+ // The natural ordering of the summaries defines their output order.
+ Set<TestSummary> summaries = Sets.newTreeSet();
+
+ int totalRun = 0; // Number of targets running at least one non-cached test.
+ int passCount = 0;
+
+ for (ConfiguredTarget testTarget : testTargets) {
+ TestSummary summary = aggregateAndReportSummary(testTarget, listener).build();
+ summaries.add(summary);
+
+ // Finished aggregating; build the final console output.
+ if (summary.actionRan()) {
+ totalRun++;
+ }
+
+ if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) {
+ passCount++;
+ }
+ }
+
+ Preconditions.checkState(summaries.size() == testTargets.size());
+
+ notifier.notify(summaries, totalRun);
+ return passCount == testTargets.size();
+ }
+
+ private static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) {
+ return status.ordinal() > other.ordinal() ? status : other;
+ }
+
+ /**
+ * Helper for differential analysis which aggregates the TestSummary
+ * for an individual target, reporting runs on the EventBus if necessary.
+ */
+ private TestSummary.Builder aggregateAndReportSummary(
+ ConfiguredTarget testTarget,
+ AggregatingTestListener listener) {
+
+ // If already reported by the listener, no work remains for this target.
+ TestSummary.Builder summary = listener.getCurrentSummary(testTarget);
+ Label testLabel = testTarget.getLabel();
+ Preconditions.checkNotNull(summary,
+ "%s did not complete test filtering, but has a test result", testLabel);
+ if (listener.targetReported(testTarget)) {
+ return summary;
+ }
+
+ Collection<Artifact> incompleteRuns = listener.getIncompleteRuns(testTarget);
+ Map<Artifact, TestResult> statusMap = listener.getStatusMap();
+
+ // We will get back multiple TestResult instances if test had to be retried several
+ // times before passing. Sharding and multiple runs of the same test without retries
+ // will be represented by separate artifacts and will produce exactly one TestResult.
+ for (Artifact testStatus : TestProvider.getTestStatusArtifacts(testTarget)) {
+ // When a build is interrupted ( eg. a broken target with --nokeep_going ) runResult could
+ // be null for an unrelated test because we were not able to even try to execute the test.
+ // In that case, for tests that were previously passing we return null ( == NO STATUS),
+ // because checking if the cached test target is up-to-date would require running the
+ // dependency checker transitively.
+ TestResult runResult = statusMap.get(testStatus);
+ boolean isIncompleteRun = incompleteRuns.contains(testStatus);
+ if (runResult == null) {
+ summary = markIncomplete(summary);
+ } else if (isIncompleteRun) {
+ // Only process results which were not recorded by the listener.
+
+ boolean newlyFetched = !statusMap.containsKey(testStatus);
+ summary = incrementalAnalyze(summary, runResult);
+ if (newlyFetched) {
+ eventBus.post(runResult);
+ }
+ Preconditions.checkState(
+ listener.getIncompleteRuns(testTarget).contains(testStatus) == isIncompleteRun,
+ "TestListener changed in differential analysis. Ensure it isn't still registered.");
+ }
+ }
+
+ // The target was not posted by the listener and must be posted now.
+ eventBus.post(summary.build());
+ return summary;
+ }
+
+ /**
+ * Incrementally updates a TestSummary given an existing summary
+ * and a new TestResult. Only call on built targets.
+ *
+ * @param summaryBuilder Existing unbuilt test summary associated with a target.
+ * @param result New test result to aggregate into the summary.
+ * @return The updated TestSummary.
+ */
+ public TestSummary.Builder incrementalAnalyze(TestSummary.Builder summaryBuilder,
+ TestResult result) {
+ // Cache retrieval should have been performed already.
+ Preconditions.checkNotNull(result);
+ Preconditions.checkNotNull(summaryBuilder);
+ TestSummary existingSummary = Preconditions.checkNotNull(summaryBuilder.peek());
+
+ TransitiveInfoCollection target = existingSummary.getTarget();
+ Preconditions.checkNotNull(
+ target, "The existing TestSummary must be associated with a target");
+
+ BlazeTestStatus status = existingSummary.getStatus();
+ int numCached = existingSummary.numCached();
+ int numLocalActionCached = existingSummary.numLocalActionCached();
+
+ if (!existingSummary.actionRan() && !result.isCached()) {
+ // At least one run of the test actually ran uncached.
+ summaryBuilder.setActionRan(true);
+
+ // Coverage data artifact will be identical for all test results - it is provided by the
+ // TestRunnerAction and all results in this collection associate with the same action.
+ PathFragment coverageData = result.getCoverageData();
+ if (coverageData != null) {
+ summaryBuilder.addCoverageFiles(
+ Collections.singletonList(execRoot.getRelative(coverageData)));
+ }
+ }
+
+ if (result.isCached() || result.getData().getRemotelyCached()) {
+ numCached++;
+ }
+ if (result.isCached()) {
+ numLocalActionCached++;
+ }
+
+ if (!executionOptions.runsPerTestDetectsFlakes) {
+ status = aggregateStatus(status, result.getData().getStatus());
+ } else {
+ int shardNumber = result.getShardNum();
+ int runsPerTestForLabel = target.getProvider(TestProvider.class).getTestParams().getRuns();
+ List<BlazeTestStatus> singleShardStatuses = summaryBuilder.addShardStatus(
+ shardNumber, result.getData().getStatus());
+ if (singleShardStatuses.size() == runsPerTestForLabel) {
+ BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS;
+ int passes = 0;
+ for (BlazeTestStatus runStatusForShard : singleShardStatuses) {
+ shardStatus = aggregateStatus(shardStatus, runStatusForShard);
+ if (TestResult.isBlazeTestStatusPassed(shardStatus)) {
+ passes++;
+ }
+ }
+ // Under the RunsPerTestDetectsFlakes option, return flaky if 1 <= p < n shards pass.
+ // If all results pass or fail, aggregate the passing/failing shardStatus.
+ if (passes == 0 || passes == runsPerTestForLabel) {
+ status = aggregateStatus(status, shardStatus);
+ } else {
+ status = aggregateStatus(status, BlazeTestStatus.FLAKY);
+ }
+ }
+ }
+
+ List<String> filtered = new ArrayList<>();
+ warningLoop: for (String warning : result.getData().getWarningList()) {
+ for (String ignoredPrefix : Constants.IGNORED_TEST_WARNING_PREFIXES) {
+ if (warning.startsWith(ignoredPrefix)) {
+ continue warningLoop;
+ }
+ }
+
+ filtered.add(warning);
+ }
+
+ List<Path> passed = new ArrayList<>();
+ if (result.getData().hasPassedLog()) {
+ passed.add(result.getTestAction().getTestLog().getPath().getRelative(
+ result.getData().getPassedLog()));
+ }
+
+ List<Path> failed = new ArrayList<>();
+ for (String path : result.getData().getFailedLogsList()) {
+ failed.add(result.getTestAction().getTestLog().getPath().getRelative(path));
+ }
+
+ summaryBuilder
+ .addTestTimes(result.getData().getTestTimesList())
+ .addPassedLogs(passed)
+ .addFailedLogs(failed)
+ .addWarnings(filtered)
+ .collectFailedTests(result.getData().getTestCase())
+ .setRanRemotely(result.getData().getIsRemoteStrategy());
+
+ List<String> warnings = new ArrayList<>();
+ if (status == BlazeTestStatus.PASSED) {
+ if (shouldEmitTestSizeWarningInSummary(
+ summaryOptions.testVerboseTimeoutWarnings,
+ warnings, result.getData().getTestProcessTimesList(), target)) {
+ summaryBuilder.setWasUnreportedWrongSize(true);
+ }
+ }
+
+ return summaryBuilder
+ .setStatus(status)
+ .setNumCached(numCached)
+ .setNumLocalActionCached(numLocalActionCached)
+ .addWarnings(warnings);
+ }
+
+ private TestSummary.Builder markIncomplete(TestSummary.Builder summaryBuilder) {
+ // TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and
+ // tests with no status and post it here.
+ TestSummary summary = summaryBuilder.peek();
+ BlazeTestStatus status = summary.getStatus();
+ if (status != BlazeTestStatus.NO_STATUS) {
+ status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE);
+ }
+
+ return summaryBuilder.setStatus(status);
+ }
+
+ TestSummary.Builder markUnbuilt(TestSummary.Builder summary, boolean blazeHalted) {
+ BlazeTestStatus runStatus = blazeHalted ? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING
+ : (executionOptions.testCheckUpToDate
+ ? BlazeTestStatus.NO_STATUS
+ : BlazeTestStatus.FAILED_TO_BUILD);
+
+ return summary.setStatus(runStatus);
+ }
+
+ /**
+ * Checks whether the specified test timeout could have been smaller and adds
+ * a warning message if verbose is true.
+ *
+ * <p>Returns true if there was a test with the wrong timeout, but if was not
+ * reported.
+ */
+ private static boolean shouldEmitTestSizeWarningInSummary(boolean verbose,
+ List<String> warnings, List<Long> testTimes, TransitiveInfoCollection target) {
+
+ TestTimeout specifiedTimeout =
+ target.getProvider(TestProvider.class).getTestParams().getTimeout();
+ long maxTimeOfShard = 0;
+
+ for (Long shardTime : testTimes) {
+ if (shardTime != null) {
+ maxTimeOfShard = Math.max(maxTimeOfShard, shardTime);
+ }
+ }
+
+ int maxTimeInSeconds = (int) (maxTimeOfShard / 1000);
+
+ if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) {
+ TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds);
+ TestSize expectedSize = TestSize.getTestSize(expectedTimeout);
+ if (verbose) {
+ StringBuilder builder = new StringBuilder(String.format(
+ "Test execution time (%.1fs excluding execution overhead) outside of "
+ + "range for %s tests. Consider setting timeout=\"%s\"",
+ maxTimeOfShard / 1000.0,
+ specifiedTimeout.prettyPrint(),
+ expectedTimeout));
+ if (expectedSize != null) {
+ builder.append(" or size=\"").append(expectedSize).append("\"");
+ }
+ builder.append(". You need not modify the size if you think it is correct.");
+ warnings.add(builder.toString());
+ return false;
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java b/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java
new file mode 100644
index 0000000..d7dbebb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java
@@ -0,0 +1,30 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import java.util.Set;
+
+/**
+ * Used to notify interested parties of test results.
+ */
+public interface TestResultNotifier {
+
+ /**
+ * @param summaries Summary of all targets that were supposed to be tested
+ * (regardless whether they actually were executed).
+ * @param numberOfExecutedTargets the number of targets that were actually run.
+ * Must not exceed summaries.size().
+ */
+ void notify(Set<TestSummary> summaries, int numberOfExecutedTargets);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java b/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java
new file mode 100644
index 0000000..171f150
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java
@@ -0,0 +1,428 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Multimap;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter.Mode;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.FailedTestCasesStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Test summary entry. Stores summary information for a single test rule.
+ * Also used to sort summary output by status.
+ *
+ * <p>Invariant:
+ * All TestSummary mutations should be performed through the Builder.
+ * No direct TestSummary methods (except the constructor) may mutate the object.
+ */
+@VisibleForTesting // Ideally package-scoped.
+public class TestSummary implements Comparable<TestSummary> {
+ /**
+ * Builder class responsible for creating and altering TestSummary objects.
+ */
+ public static class Builder {
+ private TestSummary summary;
+ private boolean built;
+
+ private Builder() {
+ summary = new TestSummary();
+ built = false;
+ }
+
+ private void mergeFrom(TestSummary existingSummary) {
+ // Yuck, manually fill in fields.
+ summary.shardRunStatuses = ArrayListMultimap.create(existingSummary.shardRunStatuses);
+ setTarget(existingSummary.target);
+ setStatus(existingSummary.status);
+ addCoverageFiles(existingSummary.coverageFiles);
+ addPassedLogs(existingSummary.passedLogs);
+ addFailedLogs(existingSummary.failedLogs);
+
+ if (existingSummary.failedTestCasesStatus != null) {
+ addFailedTestCases(existingSummary.getFailedTestCases(),
+ existingSummary.getFailedTestCasesStatus());
+ }
+
+ addTestTimes(existingSummary.testTimes);
+ addWarnings(existingSummary.warnings);
+ setActionRan(existingSummary.actionRan);
+ setNumCached(existingSummary.numCached);
+ setRanRemotely(existingSummary.ranRemotely);
+ setWasUnreportedWrongSize(existingSummary.wasUnreportedWrongSize);
+ }
+
+ // Implements copy on write logic, allowing reuse of the same builder.
+ private void checkMutation() {
+ // If mutating the builder after an object was built, create another copy.
+ if (built) {
+ built = false;
+ TestSummary lastSummary = summary;
+ summary = new TestSummary();
+ mergeFrom(lastSummary);
+ }
+ }
+
+ // This used to return a reference to the value on success.
+ // However, since it can alter the summary member, inlining it in an
+ // assignment to a property of summary was unsafe.
+ private void checkMutation(Object value) {
+ Preconditions.checkNotNull(value);
+ checkMutation();
+ }
+
+ public Builder setTarget(ConfiguredTarget target) {
+ checkMutation(target);
+ summary.target = target;
+ return this;
+ }
+
+ public Builder setStatus(BlazeTestStatus status) {
+ checkMutation(status);
+ summary.status = status;
+ return this;
+ }
+
+ public Builder addCoverageFiles(List<Path> coverageFiles) {
+ checkMutation(coverageFiles);
+ summary.coverageFiles.addAll(coverageFiles);
+ return this;
+ }
+
+ public Builder addPassedLogs(List<Path> passedLogs) {
+ checkMutation(passedLogs);
+ summary.passedLogs.addAll(passedLogs);
+ return this;
+ }
+
+ public Builder addFailedLogs(List<Path> failedLogs) {
+ checkMutation(failedLogs);
+ summary.failedLogs.addAll(failedLogs);
+ return this;
+ }
+
+ public Builder collectFailedTests(TestCase testCase) {
+ if (testCase == null) {
+ summary.failedTestCasesStatus = FailedTestCasesStatus.NOT_AVAILABLE;
+ return this;
+ }
+ summary.failedTestCasesStatus = FailedTestCasesStatus.FULL;
+ return collectFailedTestCases(testCase);
+ }
+
+ private Builder collectFailedTestCases(TestCase testCase) {
+ if (testCase.getChildCount() > 0) {
+ // This is a non-leaf result. Traverse its children, but do not add its
+ // name to the output list. It should not contain any 'failure' or
+ // 'error' tags, but we want to be lax here, because the syntax of the
+ // test.xml file is also lax.
+ for (TestCase child : testCase.getChildList()) {
+ collectFailedTestCases(child);
+ }
+ } else {
+ // This is a leaf result. If it passed, don't add it.
+ if (testCase.getStatus() == TestCase.Status.PASSED) {
+ return this;
+ }
+
+ String name = testCase.getName();
+ String className = testCase.getClassName();
+ if (name == null || className == null) {
+ // A test case detail is not really interesting if we cannot tell which
+ // one it is.
+ this.summary.failedTestCasesStatus = FailedTestCasesStatus.PARTIAL;
+ return this;
+ }
+
+ this.summary.failedTestCases.add(testCase);
+ }
+ return this;
+ }
+
+ public Builder addFailedTestCases(List<TestCase> testCases, FailedTestCasesStatus status) {
+ checkMutation(status);
+ checkMutation(testCases);
+
+ if (summary.failedTestCasesStatus == null) {
+ summary.failedTestCasesStatus = status;
+ } else if (summary.failedTestCasesStatus != status) {
+ summary.failedTestCasesStatus = FailedTestCasesStatus.PARTIAL;
+ }
+
+ if (testCases.isEmpty()) {
+ return this;
+ }
+
+ // union of summary.failedTestCases, testCases
+ Map<String, TestCase> allCases = new TreeMap<>();
+ if (summary.failedTestCases != null) {
+ for (TestCase detail : summary.failedTestCases) {
+ allCases.put(detail.getClassName() + "." + detail.getName(), detail);
+ }
+ }
+ for (TestCase detail : testCases) {
+ allCases.put(detail.getClassName() + "." + detail.getName(), detail);
+ }
+
+ summary.failedTestCases = new ArrayList<TestCase>(allCases.values());
+ return this;
+ }
+
+ public Builder addTestTimes(List<Long> testTimes) {
+ checkMutation(testTimes);
+ summary.testTimes.addAll(testTimes);
+ return this;
+ }
+
+ public Builder addWarnings(List<String> warnings) {
+ checkMutation(warnings);
+ summary.warnings.addAll(warnings);
+ return this;
+ }
+
+ public Builder setActionRan(boolean actionRan) {
+ checkMutation();
+ summary.actionRan = actionRan;
+ return this;
+ }
+
+ public Builder setNumCached(int numCached) {
+ checkMutation();
+ summary.numCached = numCached;
+ return this;
+ }
+
+ public Builder setNumLocalActionCached(int numLocalActionCached) {
+ checkMutation();
+ summary.numLocalActionCached = numLocalActionCached;
+ return this;
+ }
+
+ public Builder setRanRemotely(boolean ranRemotely) {
+ checkMutation();
+ summary.ranRemotely = ranRemotely;
+ return this;
+ }
+
+ public Builder setWasUnreportedWrongSize(boolean wasUnreportedWrongSize) {
+ checkMutation();
+ summary.wasUnreportedWrongSize = wasUnreportedWrongSize;
+ return this;
+ }
+
+ /**
+ * Records a new result for the given shard of the test.
+ *
+ * @return an immutable view of the statuses associated with the shard, with the new element.
+ */
+ public List<BlazeTestStatus> addShardStatus(int shardNumber, BlazeTestStatus status) {
+ Preconditions.checkState(summary.shardRunStatuses.put(shardNumber, status),
+ "shardRunStatuses must allow duplicate statuses");
+ return ImmutableList.copyOf(summary.shardRunStatuses.get(shardNumber));
+ }
+
+ /**
+ * Returns the created TestSummary object.
+ * Any actions following a build() will create another copy of the same values.
+ * Since no mutators are provided directly by TestSummary, a copy will not
+ * be produced if two builds are invoked in a row without calling a setter.
+ */
+ public TestSummary build() {
+ peek();
+ if (!built) {
+ makeSummaryImmutable();
+ // else: it is already immutable.
+ }
+ Preconditions.checkState(built, "Built flag was not set");
+ return summary;
+ }
+
+ /**
+ * Within-package, it is possible to read directly from an
+ * incompletely-built TestSummary. Used to pass Builders around directly.
+ */
+ TestSummary peek() {
+ Preconditions.checkNotNull(summary.target, "Target cannot be null");
+ Preconditions.checkNotNull(summary.status, "Status cannot be null");
+ return summary;
+ }
+
+ private void makeSummaryImmutable() {
+ // Once finalized, the list types are immutable.
+ summary.passedLogs = Collections.unmodifiableList(summary.passedLogs);
+ summary.failedLogs = Collections.unmodifiableList(summary.failedLogs);
+ summary.warnings = Collections.unmodifiableList(summary.warnings);
+ summary.coverageFiles = Collections.unmodifiableList(summary.coverageFiles);
+ summary.testTimes = Collections.unmodifiableList(summary.testTimes);
+
+ built = true;
+ }
+ }
+
+ private ConfiguredTarget target;
+ private BlazeTestStatus status;
+ // Currently only populated if --runs_per_test_detects_flakes is enabled.
+ private Multimap<Integer, BlazeTestStatus> shardRunStatuses = ArrayListMultimap.create();
+ private int numCached;
+ private int numLocalActionCached;
+ private boolean actionRan;
+ private boolean ranRemotely;
+ private boolean wasUnreportedWrongSize;
+ private List<TestCase> failedTestCases = new ArrayList<>();
+ private List<Path> passedLogs = new ArrayList<>();
+ private List<Path> failedLogs = new ArrayList<>();
+ private List<String> warnings = new ArrayList<>();
+ private List<Path> coverageFiles = new ArrayList<>();
+ private List<Long> testTimes = new ArrayList<>();
+ private FailedTestCasesStatus failedTestCasesStatus = null;
+
+ // Don't allow public instantiation; go through the Builder.
+ private TestSummary() {
+ }
+
+ /**
+ * Creates a new Builder allowing construction of a new TestSummary object.
+ */
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ /**
+ * Creates a new Builder initialized with a copy of the existing object's values.
+ */
+ public static Builder newBuilderFromExisting(TestSummary existing) {
+ Builder builder = new Builder();
+ builder.mergeFrom(existing);
+ return builder;
+ }
+
+ public ConfiguredTarget getTarget() {
+ return target;
+ }
+
+ public BlazeTestStatus getStatus() {
+ return status;
+ }
+
+ public boolean isCached() {
+ return numCached > 0;
+ }
+
+ public boolean isLocalActionCached() {
+ return numLocalActionCached > 0;
+ }
+
+ public int numLocalActionCached() {
+ return numLocalActionCached;
+ }
+
+ public int numCached() {
+ return numCached;
+ }
+
+ private int numUncached() {
+ return totalRuns() - numCached;
+ }
+
+ public boolean actionRan() {
+ return actionRan;
+ }
+
+ public boolean ranRemotely() {
+ return ranRemotely;
+ }
+
+ public boolean wasUnreportedWrongSize() {
+ return wasUnreportedWrongSize;
+ }
+
+ public List<TestCase> getFailedTestCases() {
+ return failedTestCases;
+ }
+
+ public List<Path> getCoverageFiles() {
+ return coverageFiles;
+ }
+
+ public List<Path> getPassedLogs() {
+ return passedLogs;
+ }
+
+ public List<Path> getFailedLogs() {
+ return failedLogs;
+ }
+
+ public FailedTestCasesStatus getFailedTestCasesStatus() {
+ return failedTestCasesStatus;
+ }
+
+ /**
+ * Returns an immutable view of the warnings associated with this test.
+ */
+ public List<String> getWarnings() {
+ return Collections.unmodifiableList(warnings);
+ }
+
+ private static int getSortKey(BlazeTestStatus status) {
+ return status == BlazeTestStatus.PASSED ? -1 : status.ordinal();
+ }
+
+ @Override
+ public int compareTo(TestSummary that) {
+ if (this.isCached() != that.isCached()) {
+ return this.isCached() ? -1 : 1;
+ } else if ((this.isCached() && that.isCached()) && (this.numUncached() != that.numUncached())) {
+ return this.numUncached() - that.numUncached();
+ } else if (this.status != that.status) {
+ return getSortKey(this.status) - getSortKey(that.status);
+ } else {
+ Artifact thisExecutable = this.target.getProvider(FilesToRunProvider.class).getExecutable();
+ Artifact thatExecutable = that.target.getProvider(FilesToRunProvider.class).getExecutable();
+ return thisExecutable.getPath().compareTo(thatExecutable.getPath());
+ }
+ }
+
+ public List<Long> getTestTimes() {
+ // The return result is unmodifiable (UnmodifiableList instance)
+ return testTimes;
+ }
+
+ public int getNumCached() {
+ return numCached;
+ }
+
+ public int totalRuns() {
+ return testTimes.size();
+ }
+
+ static Mode getStatusMode(BlazeTestStatus status) {
+ return status == BlazeTestStatus.PASSED
+ ? Mode.INFO
+ : (status == BlazeTestStatus.FLAKY ? Mode.WARNING : Mode.ERROR);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java b/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java
new file mode 100644
index 0000000..91c1488
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java
@@ -0,0 +1,255 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.devtools.build.lib.rules.test.TestLogHelper;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat;
+import com.google.devtools.build.lib.util.LoggingUtil;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter.Mode;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.FailedTestCasesStatus;
+import com.google.devtools.build.lib.view.test.TestStatus.TestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+
+/**
+ * Print test statistics in human readable form.
+ */
+public class TestSummaryPrinter {
+
+ /**
+ * Print the cached test log to the given printer.
+ */
+ public static void printCachedOutput(TestSummary summary,
+ TestOutputFormat testOutput,
+ AnsiTerminalPrinter printer) {
+
+ String testName = summary.getTarget().getLabel().toString();
+ List<String> allLogs = new ArrayList<>();
+ for (Path path : summary.getFailedLogs()) {
+ allLogs.add(path.getPathString());
+ }
+ for (Path path : summary.getPassedLogs()) {
+ allLogs.add(path.getPathString());
+ }
+ printer.printLn("" + TestSummary.getStatusMode(summary.getStatus()) + summary.getStatus() + ": "
+ + Mode.DEFAULT + testName + " (see " + Joiner.on(' ').join(allLogs) + ")");
+ printer.printLn(Mode.INFO + "INFO: " + Mode.DEFAULT + "From Testing " + testName);
+
+ // Whether to output the target at all was checked by the caller.
+ // Now check whether to output failing shards.
+ if (TestLogHelper.shouldOutputTestLog(testOutput, false)) {
+ for (Path path : summary.getFailedLogs()) {
+ try {
+ TestLogHelper.writeTestLog(path, testName, printer.getOutputStream());
+ } catch (IOException e) {
+ printer.printLn("==================== Could not read test output for " + testName);
+ LoggingUtil.logToRemote(Level.WARNING, "Error while reading test log", e);
+ }
+ }
+ }
+
+ // And passing shards, independently.
+ if (TestLogHelper.shouldOutputTestLog(testOutput, true)) {
+ for (Path path : summary.getPassedLogs()) {
+ try {
+ TestLogHelper.writeTestLog(path, testName, printer.getOutputStream());
+ } catch (Exception e) {
+ printer.printLn("==================== Could not read test output for " + testName);
+ LoggingUtil.logToRemote(Level.WARNING, "Error while reading test log", e);
+ }
+ }
+ }
+ }
+
+ private static String statusString(BlazeTestStatus status) {
+ return status.toString().replace('_', ' ');
+ }
+
+ /**
+ * Prints summary status for a single test.
+ * @param terminalPrinter The printer to print to
+ */
+ public static void print(
+ TestSummary summary,
+ AnsiTerminalPrinter terminalPrinter,
+ boolean verboseSummary, boolean printFailedTestCases) {
+ // Skip output for tests that failed to build.
+ if (summary.getStatus() == BlazeTestStatus.FAILED_TO_BUILD) {
+ return;
+ }
+ String message = getCacheMessage(summary) + statusString(summary.getStatus());
+ terminalPrinter.print(
+ Strings.padEnd(summary.getTarget().getLabel().toString(), 78 - message.length(), ' ')
+ + " " + TestSummary.getStatusMode(summary.getStatus()) + message + Mode.DEFAULT
+ + (verboseSummary ? getAttemptSummary(summary) + getTimeSummary(summary) : "") + "\n");
+
+ if (printFailedTestCases && summary.getStatus() == BlazeTestStatus.FAILED) {
+ if (summary.getFailedTestCasesStatus() == FailedTestCasesStatus.NOT_AVAILABLE) {
+ terminalPrinter.print(
+ Mode.WARNING + " (individual test case information not available) "
+ + Mode.DEFAULT + "\n");
+ } else {
+ for (TestCase testCase : summary.getFailedTestCases()) {
+ if (testCase.getStatus() != TestCase.Status.PASSED) {
+ TestSummaryPrinter.printTestCase(terminalPrinter, testCase);
+ }
+ }
+
+ if (summary.getFailedTestCasesStatus() != FailedTestCasesStatus.FULL) {
+ terminalPrinter.print(
+ Mode.WARNING
+ + " (some shards did not report details, list of failed test"
+ + " cases incomplete)\n"
+ + Mode.DEFAULT);
+ }
+ }
+ }
+
+ if (printFailedTestCases) {
+ // In this mode, test output and coverage files would just clutter up
+ // the output.
+ return;
+ }
+
+ for (String warning : summary.getWarnings()) {
+ terminalPrinter.print(" " + AnsiTerminalPrinter.Mode.WARNING + "WARNING: "
+ + AnsiTerminalPrinter.Mode.DEFAULT + warning + "\n");
+ }
+
+ for (Path path : summary.getFailedLogs()) {
+ if (path.exists()) {
+ // Don't use getPrettyPath() here - we want to print the absolute path,
+ // so that it cut and paste into a different terminal, and we don't
+ // want to use the blaze-bin etc. symbolic links because they could be changed
+ // by a subsequent build with different options.
+ terminalPrinter.print(" " + path.getPathString() + "\n");
+ }
+ }
+ for (Path path : summary.getCoverageFiles()) {
+ // Print only non-trivial coverage files.
+ try {
+ if (path.exists() && path.getFileSize() > 0) {
+ terminalPrinter.print(" " + path.getPathString() + "\n");
+ }
+ } catch (IOException e) {
+ LoggingUtil.logToRemote(Level.WARNING, "Error while reading coverage data file size",
+ e);
+ }
+ }
+ }
+
+ /**
+ * Prints the result of an individual test case. It is assumed not to have
+ * passed, since passed test cases are not reported.
+ */
+ static void printTestCase(
+ AnsiTerminalPrinter terminalPrinter, TestCase testCase) {
+ String timeSummary;
+ if (testCase.hasRunDurationMillis()) {
+ timeSummary = " ("
+ + timeInSec(testCase.getRunDurationMillis(), TimeUnit.MILLISECONDS)
+ + ")";
+ } else {
+ timeSummary = "";
+ }
+
+ terminalPrinter.print(
+ " "
+ + Mode.ERROR
+ + Strings.padEnd(testCase.getStatus().toString(), 8, ' ')
+ + Mode.DEFAULT
+ + testCase.getClassName()
+ + "."
+ + testCase.getName()
+ + timeSummary
+ + "\n");
+ }
+
+ /**
+ * Return the given time in seconds, to 1 decimal place,
+ * i.e. "32.1s".
+ */
+ static String timeInSec(long time, TimeUnit unit) {
+ double ms = TimeUnit.MILLISECONDS.convert(time, unit);
+ return String.format("%.1fs", ms / 1000.0);
+ }
+
+ static String getAttemptSummary(TestSummary summary) {
+ int attempts = summary.getPassedLogs().size() + summary.getFailedLogs().size();
+ if (attempts > 1) {
+ // Print number of failed runs for failed tests if testing was completed.
+ if (summary.getStatus() == BlazeTestStatus.FLAKY) {
+ return ", failed in " + summary.getFailedLogs().size() + " out of " + attempts;
+ }
+ if (summary.getStatus() == BlazeTestStatus.TIMEOUT
+ || summary.getStatus() == BlazeTestStatus.FAILED) {
+ return " in " + summary.getFailedLogs().size() + " out of " + attempts;
+ }
+ }
+ return "";
+ }
+
+ static String getCacheMessage(TestSummary summary) {
+ if (summary.getNumCached() == 0 || summary.getStatus() == BlazeTestStatus.INCOMPLETE) {
+ return "";
+ } else if (summary.getNumCached() == summary.totalRuns()) {
+ return "(cached) ";
+ } else {
+ return String.format("(%d/%d cached) ", summary.getNumCached(), summary.totalRuns());
+ }
+ }
+
+ static String getTimeSummary(TestSummary summary) {
+ if (summary.getTestTimes().isEmpty()) {
+ return "";
+ } else if (summary.getTestTimes().size() == 1) {
+ return " in " + timeInSec(summary.getTestTimes().get(0), TimeUnit.MILLISECONDS);
+ } else {
+ // We previously used com.google.math for this, which added about 1 MB of deps to the total
+ // size. If we re-introduce a dependency on that package, we could revert this change.
+ long min = summary.getTestTimes().get(0).longValue(), max = min, sum = 0;
+ double sumOfSquares = 0.0;
+ for (Long l : summary.getTestTimes()) {
+ long value = l.longValue();
+ min = value < min ? value : min;
+ max = value > max ? value : max;
+ sum += value;
+ sumOfSquares += ((double) value) * (double) value;
+ }
+ double mean = ((double) sum) / summary.getTestTimes().size();
+ double stddev = Math.sqrt((sumOfSquares - sum * mean) / summary.getTestTimes().size());
+ // For sharded tests, we print the max time on the same line as
+ // the test, and then print more detailed info about the
+ // distribution of times on the next line.
+ String maxTime = timeInSec(max, TimeUnit.MILLISECONDS);
+ return String.format(
+ " in %s\n Stats over %d runs: max = %s, min = %s, avg = %s, dev = %s",
+ maxTime,
+ summary.getTestTimes().size(),
+ maxTime,
+ timeInSec(min, TimeUnit.MILLISECONDS),
+ timeInSec((long) mean, TimeUnit.MILLISECONDS),
+ timeInSec((long) stddev, TimeUnit.MILLISECONDS));
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java
new file mode 100644
index 0000000..d6f61eb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java
@@ -0,0 +1,69 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.analysis.BuildView;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.List;
+
+/**
+ * Handles the 'build' command on the Blaze command line, including targets
+ * named by arguments passed to Blaze.
+ */
+@Command(name = "build",
+ builds = true,
+ options = { BuildRequestOptions.class,
+ ExecutionOptions.class,
+ PackageCacheOptions.class,
+ BuildView.Options.class,
+ LoadingPhaseRunner.Options.class,
+ BuildConfiguration.Options.class,
+ },
+ usesConfigurationOptions = true,
+ shortDescription = "Builds the specified targets.",
+ allowResidue = true,
+ help = "resource:build.txt")
+public final class BuildCommand implements BlazeCommand {
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser)
+ throws AbruptExitException {
+ ProjectFileSupport.handleProjectFiles(runtime, optionsParser, "build");
+ }
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ List<String> targets = ProjectFileSupport.getTargets(runtime, options);
+
+ BuildRequest request = BuildRequest.create(
+ getClass().getAnnotation(Command.class).name(), options,
+ runtime.getStartupOptionsProvider(),
+ targets,
+ runtime.getReporter().getOutErr(), runtime.getCommandId(), runtime.getCommandStartTime());
+ return runtime.getBuildTool().processRequest(request, null).getExitCondition();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java
new file mode 100644
index 0000000..0bb5a0e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java
@@ -0,0 +1,95 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandUtils;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * The 'blaze canonicalize-flags' command.
+ */
+@Command(name = "canonicalize-flags",
+ options = { CanonicalizeCommand.Options.class },
+ allowResidue = true,
+ mustRunInWorkspace = false,
+ shortDescription = "Canonicalizes a list of Blaze options.",
+ help = "This command canonicalizes a list of Blaze options. Don't forget to prepend '--' "
+ + "to end option parsing before the flags to canonicalize.\n"
+ + "%{options}")
+public final class CanonicalizeCommand implements BlazeCommand {
+
+ public static class CommandConverter implements Converter<String> {
+
+ @Override
+ public String convert(String input) throws OptionsParsingException {
+ if (input.equals("build")) {
+ return input;
+ } else if (input.equals("test")) {
+ return input;
+ }
+ throw new OptionsParsingException("Not a valid command: '" + input + "' (should be "
+ + getTypeDescription() + ")");
+ }
+
+ @Override
+ public String getTypeDescription() {
+ return "build or test";
+ }
+ }
+
+ public static class Options extends OptionsBase {
+
+ @Option(name = "for_command",
+ defaultValue = "build",
+ category = "misc",
+ converter = CommandConverter.class,
+ help = "The command for which the options should be canonicalized.")
+ public String forCommand;
+ }
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ BlazeCommand command = runtime.getCommandMap().get(
+ options.getOptions(Options.class).forCommand);
+ Collection<Class<? extends OptionsBase>> optionsClasses =
+ BlazeCommandUtils.getOptions(
+ command.getClass(), runtime.getBlazeModules(), runtime.getRuleClassProvider());
+ try {
+ List<String> result = OptionsParser.canonicalize(optionsClasses, options.getResidue());
+ for (String piece : result) {
+ runtime.getReporter().getOutErr().printOutLn(piece);
+ }
+ } catch (OptionsParsingException e) {
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ return ExitCode.SUCCESS;
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java
new file mode 100644
index 0000000..3fd300e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java
@@ -0,0 +1,185 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.util.CommandBuilder;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.ProcessUtils;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+
+/**
+ * Implements 'blaze clean'.
+ */
+@Command(name = "clean",
+ builds = true, // Does not, but people expect build options to be there
+ options = { CleanCommand.Options.class },
+ help = "resource:clean.txt",
+ shortDescription = "Removes output files and optionally stops the server.",
+ // TODO(bazel-team): Remove this - we inherit a huge number of unused options.
+ inherits = { BuildCommand.class })
+public final class CleanCommand implements BlazeCommand {
+
+ /**
+ * An interface for special options for the clean command.
+ */
+ public static class Options extends OptionsBase {
+ @Option(name = "clean_style",
+ defaultValue = "",
+ category = "clean",
+ help = "Can be either 'expunge' or 'expunge_async'.")
+ public String cleanStyle;
+
+ @Option(name = "expunge",
+ defaultValue = "false",
+ category = "clean",
+ expansion = "--clean_style=expunge",
+ help = "If specified, clean will remove the entire working tree for this Blaze " +
+ "instance, which includes all Blaze-created temporary and build output " +
+ "files, and it will stop the Blaze server if it is running.")
+ public boolean expunge;
+
+ @Option(name = "expunge_async",
+ defaultValue = "false",
+ category = "clean",
+ expansion = "--clean_style=expunge_async",
+ help = "If specified, clean will asynchronously remove the entire working tree for " +
+ "this Blaze instance, which includes all Blaze-created temporary and build " +
+ "output files, and it will stop the Blaze server if it is running. When this " +
+ "command completes, it will be safe to execute new commands in the same client, " +
+ "even though the deletion may continue in the background.")
+ public boolean expunge_async;
+ }
+
+ private static Logger LOG = Logger.getLogger(CleanCommand.class.getName());
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options)
+ throws ShutdownBlazeServerException {
+ Options cleanOptions = options.getOptions(Options.class);
+ cleanOptions.expunge_async = cleanOptions.cleanStyle.equals("expunge_async");
+ cleanOptions.expunge = cleanOptions.cleanStyle.equals("expunge");
+
+ if (cleanOptions.expunge == false && cleanOptions.expunge_async == false &&
+ !cleanOptions.cleanStyle.isEmpty()) {
+ runtime.getReporter().handle(Event.error(
+ null, "Invalid clean_style value '" + cleanOptions.cleanStyle + "'"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ String cleanBanner = cleanOptions.expunge_async ?
+ "Starting clean." :
+ "Starting clean (this may take a while). " +
+ "Consider using --expunge_async if the clean takes more than several minutes.";
+
+ runtime.getReporter().handle(Event.info(null/*location*/, cleanBanner));
+ try {
+ String symlinkPrefix =
+ options.getOptions(BuildRequest.BuildRequestOptions.class).symlinkPrefix;
+ actuallyClean(runtime, runtime.getOutputBase(), cleanOptions, symlinkPrefix);
+ return ExitCode.SUCCESS;
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
+ } catch (CommandException e) {
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ return ExitCode.RUN_FAILURE;
+ } catch (ExecException e) {
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ return ExitCode.RUN_FAILURE;
+ } catch (InterruptedException e) {
+ runtime.getReporter().handle(Event.error("clean interrupted"));
+ return ExitCode.INTERRUPTED;
+ }
+ }
+
+ private void actuallyClean(BlazeRuntime runtime,
+ Path outputBase, Options cleanOptions, String symlinkPrefix) throws IOException,
+ ShutdownBlazeServerException, CommandException, ExecException, InterruptedException {
+ if (runtime.getOutputService() != null) {
+ runtime.getOutputService().clean();
+ }
+ if (cleanOptions.expunge) {
+ LOG.info("Expunging...");
+ // Delete the big subdirectories with the important content first--this
+ // will take the most time. Then quickly delete the little locks, logs
+ // and links right before we exit. Once the lock file is gone there will
+ // be a small possibility of a server race if a client is waiting, but
+ // all significant files will be gone by then.
+ FileSystemUtils.deleteTreesBelow(outputBase);
+ FileSystemUtils.deleteTree(outputBase);
+ } else if (cleanOptions.expunge_async) {
+ LOG.info("Expunging asynchronously...");
+ String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid();
+
+ // Keeping tempOutputBase in the same directory ensures it remains in the
+ // same file system, and therefore the mv will be atomic and fast.
+ Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName);
+ outputBase.renameTo(tempOutputBase);
+ runtime.getReporter().handle(Event.info(
+ null, "Output base moved to " + tempOutputBase + " for deletion"));
+
+ // Daemonize the shell and use the double-fork idiom to ensure that the shell
+ // exits even while the "rm -rf" command continues.
+ String command = String.format("exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&",
+ ShellEscaper.escapeString(tempOutputBase.getPathString()));
+
+ LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command));
+
+ // Doesn't throw iff command exited and was successful.
+ new CommandBuilder().addArg(command).useShell(true)
+ .setWorkingDir(tempOutputBase.getParentDirectory())
+ .build().execute();
+ } else {
+ LOG.info("Output cleaning...");
+ runtime.clearCaches();
+ for (String directory : new String[] {
+ BlazeDirectories.RELATIVE_OUTPUT_PATH, runtime.getWorkspaceName() }) {
+ Path child = outputBase.getChild(directory);
+ if (child.exists()) {
+ LOG.finest("Cleaning " + child);
+ FileSystemUtils.deleteTreesBelow(child);
+ }
+ }
+ }
+ // remove convenience links
+ OutputDirectoryLinksUtils.removeOutputDirectoryLinks(
+ runtime.getWorkspaceName(), runtime.getWorkspace(), runtime.getReporter(), symlinkPrefix);
+ // shutdown on expunge cleans
+ if (cleanOptions.expunge || cleanOptions.expunge_async) {
+ throw new ShutdownBlazeServerException(0);
+ }
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java
new file mode 100644
index 0000000..5267e71
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java
@@ -0,0 +1,248 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.docgen.BlazeRuleHelpPrinter;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandUtils;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The 'blaze help' command, which prints all available commands as well as
+ * specific help pages.
+ */
+@Command(name = "help",
+ options = { HelpCommand.Options.class },
+ allowResidue = true,
+ mustRunInWorkspace = false,
+ shortDescription = "Prints help for commands, or the index.",
+ help = "resource:help.txt")
+public final class HelpCommand implements BlazeCommand {
+ public static class Options extends OptionsBase {
+
+ @Option(name = "help_verbosity",
+ category = "help",
+ defaultValue = "medium",
+ converter = Converters.HelpVerbosityConverter.class,
+ help = "Select the verbosity of the help command.")
+ public OptionsParser.HelpVerbosity helpVerbosity;
+
+ @Option(name = "long",
+ abbrev = 'l',
+ defaultValue = "null",
+ category = "help",
+ expansion = {"--help_verbosity", "long"},
+ help = "Show full description of each option, instead of just its name.")
+ public Void showLongFormOptions;
+
+ @Option(name = "short",
+ defaultValue = "null",
+ category = "help",
+ expansion = {"--help_verbosity", "short"},
+ help = "Show only the names of the options, not their types or meanings.")
+ public Void showShortFormOptions;
+ }
+
+ /**
+ * Returns a map that maps option categories to descriptive help strings for categories that
+ * are not part of the Bazel core.
+ */
+ private ImmutableMap<String, String> getOptionCategories(BlazeRuntime runtime) {
+ ImmutableMap.Builder<String, String> optionCategoriesBuilder = ImmutableMap.builder();
+ optionCategoriesBuilder
+ .put("checking",
+ "Checking options, which control Blaze's error checking and/or warnings")
+ .put("coverage",
+ "Options that affect how Blaze generates code coverage information")
+ .put("experimental",
+ "Experimental options, which control experimental (and potentially risky) features")
+ .put("flags",
+ "Flags options, for passing options to other tools")
+ .put("help",
+ "Help options")
+ .put("host jvm startup",
+ "Options that affect the startup of the Blaze server's JVM")
+ .put("misc",
+ "Miscellaneous options")
+ .put("package loading",
+ "Options that specify how to locate packages")
+ .put("query",
+ "Options affecting the 'blaze query' dependency query command")
+ .put("run",
+ "Options specific to 'blaze run'")
+ .put("semantics",
+ "Semantics options, which affect the build commands and/or output file contents")
+ .put("server startup",
+ "Startup options, which affect the startup of the Blaze server")
+ .put("strategy",
+ "Strategy options, which affect how Blaze will execute the build")
+ .put("testing",
+ "Options that affect how Blaze runs tests")
+ .put("verbosity",
+ "Verbosity options, which control what Blaze prints")
+ .put("version",
+ "Version options, for selecting which version of other tools will be used")
+ .put("what",
+ "Output selection options, for determining what to build/test");
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ optionCategoriesBuilder.putAll(module.getOptionCategories());
+ }
+ return optionCategoriesBuilder.build();
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ OutErr outErr = runtime.getReporter().getOutErr();
+ Options helpOptions = options.getOptions(Options.class);
+ if (options.getResidue().isEmpty()) {
+ emitBlazeVersionInfo(outErr);
+ emitGenericHelp(runtime, outErr);
+ return ExitCode.SUCCESS;
+ }
+ if (options.getResidue().size() != 1) {
+ runtime.getReporter().handle(Event.error("You must specify exactly one command"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ String helpSubject = options.getResidue().get(0);
+ if (helpSubject.equals("startup_options")) {
+ emitBlazeVersionInfo(outErr);
+ emitStartupOptions(outErr, helpOptions.helpVerbosity, runtime, getOptionCategories(runtime));
+ return ExitCode.SUCCESS;
+ } else if (helpSubject.equals("target-syntax")) {
+ emitBlazeVersionInfo(outErr);
+ emitTargetSyntaxHelp(outErr, getOptionCategories(runtime));
+ return ExitCode.SUCCESS;
+ } else if (helpSubject.equals("info-keys")) {
+ emitInfoKeysHelp(runtime, outErr);
+ return ExitCode.SUCCESS;
+ }
+
+ BlazeCommand command = runtime.getCommandMap().get(helpSubject);
+ if (command == null) {
+ ConfiguredRuleClassProvider provider = runtime.getRuleClassProvider();
+ RuleClass ruleClass = provider.getRuleClassMap().get(helpSubject);
+ if (ruleClass != null && ruleClass.isDocumented()) {
+ // There is a rule with a corresponding name
+ outErr.printOut(BlazeRuleHelpPrinter.getRuleDoc(helpSubject, provider));
+ return ExitCode.SUCCESS;
+ } else {
+ runtime.getReporter().handle(Event.error(
+ null, "'" + helpSubject + "' is neither a command nor a build rule"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ }
+ emitBlazeVersionInfo(outErr);
+ outErr.printOut(BlazeCommandUtils.getUsage(
+ command.getClass(),
+ getOptionCategories(runtime),
+ helpOptions.helpVerbosity,
+ runtime.getBlazeModules(),
+ runtime.getRuleClassProvider()));
+ return ExitCode.SUCCESS;
+ }
+
+ private void emitBlazeVersionInfo(OutErr outErr) {
+ String releaseInfo = BlazeVersionInfo.instance().getReleaseName();
+ String line = "[Blaze " + releaseInfo + "]";
+ outErr.printOut(String.format("%80s\n", line));
+ }
+
+ @SuppressWarnings("unchecked") // varargs generic array creation
+ private void emitStartupOptions(OutErr outErr, OptionsParser.HelpVerbosity helpVerbosity,
+ BlazeRuntime runtime, ImmutableMap<String, String> optionCategories) {
+ outErr.printOut(
+ BlazeCommandUtils.expandHelpTopic("startup_options",
+ "resource:startup_options.txt",
+ getClass(),
+ BlazeCommandUtils.getStartupOptions(runtime.getBlazeModules()),
+ optionCategories,
+ helpVerbosity));
+ }
+
+ private void emitTargetSyntaxHelp(OutErr outErr, ImmutableMap<String, String> optionCategories) {
+ outErr.printOut(BlazeCommandUtils.expandHelpTopic("target-syntax",
+ "resource:target-syntax.txt",
+ getClass(),
+ ImmutableList.<Class<? extends OptionsBase>>of(),
+ optionCategories,
+ OptionsParser.HelpVerbosity.MEDIUM));
+ }
+
+ private void emitInfoKeysHelp(BlazeRuntime runtime, OutErr outErr) {
+ for (InfoKey key : InfoKey.values()) {
+ outErr.printOut(String.format("%-23s %s\n", key.getName(), key.getDescription()));
+ }
+
+ for (BlazeModule.InfoItem item : InfoCommand.getInfoItemMap(runtime,
+ OptionsParser.newOptionsParser(
+ ImmutableList.<Class<? extends OptionsBase>>of())).values()) {
+ outErr.printOut(String.format("%-23s %s\n", item.getName(), item.getDescription()));
+ }
+ }
+
+ private void emitGenericHelp(BlazeRuntime runtime, OutErr outErr) {
+ outErr.printOut("Usage: blaze <command> <options> ...\n\n");
+
+ outErr.printOut("Available commands:\n");
+
+ Map<String, BlazeCommand> commandsByName = runtime.getCommandMap();
+ List<String> namesInOrder = new ArrayList<>(commandsByName.keySet());
+ Collections.sort(namesInOrder);
+
+ for (String name : namesInOrder) {
+ BlazeCommand command = commandsByName.get(name);
+ Command annotation = command.getClass().getAnnotation(Command.class);
+ if (annotation.hidden()) {
+ continue;
+ }
+
+ String shortDescription = annotation.shortDescription();
+ outErr.printOut(String.format(" %-19s %s\n", name, shortDescription));
+ }
+
+ outErr.printOut("\n");
+ outErr.printOut("Getting more help:\n");
+ outErr.printOut(" blaze help <command>\n");
+ outErr.printOut(" Prints help and options for <command>.\n");
+ outErr.printOut(" blaze help startup_options\n");
+ outErr.printOut(" Options for the JVM hosting Blaze.\n");
+ outErr.printOut(" blaze help target-syntax\n");
+ outErr.printOut(" Explains the syntax for specifying targets.\n");
+ outErr.printOut(" blaze help info-keys\n");
+ outErr.printOut(" Displays a list of keys used by the info command.\n");
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java
new file mode 100644
index 0000000..31aaeb1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java
@@ -0,0 +1,448 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.Constants;
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.Attribute;
+import com.google.devtools.build.lib.packages.ProtoUtils;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build.AllowedRuleClassInfo;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build.AttributeDefinition;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build.BuildLanguage;
+import com.google.devtools.build.lib.query2.proto.proto2api.Build.RuleDefinition;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.OsUtils;
+import com.google.devtools.build.lib.util.StringUtilities;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Implementation of 'blaze info'.
+ */
+@Command(name = "info",
+ // TODO(bazel-team): this is not really a build command, but needs access to the
+ // configuration options to do its job
+ builds = true,
+ allowResidue = true,
+ binaryStdOut = true,
+ help = "resource:info.txt",
+ shortDescription = "Displays runtime info about the blaze server.",
+ options = { InfoCommand.Options.class },
+ // We have InfoCommand inherit from {@link BuildCommand} because we want all
+ // configuration defaults specified in ~/.blazerc for {@code build} to apply to
+ // {@code info} too, even though it doesn't actually do a build.
+ //
+ // (Ideally there would be a way to make {@code info} inherit just the bare
+ // minimum of relevant options from {@code build}, i.e. those that affect the
+ // values it prints. But there's no such mechanism.)
+ inherits = { BuildCommand.class })
+public class InfoCommand implements BlazeCommand {
+
+ public static class Options extends OptionsBase {
+ @Option(name = "show_make_env",
+ defaultValue = "false",
+ category = "misc",
+ help = "Include the \"Make\" environment in the output.")
+ public boolean showMakeEnvironment;
+ }
+
+ /**
+ * Unchecked variant of ExitCausingException. Below, we need to throw from the Supplier interface,
+ * which does not allow checked exceptions.
+ */
+ public static class ExitCausingRuntimeException extends RuntimeException {
+
+ private final ExitCode exitCode;
+
+ public ExitCausingRuntimeException(String message, ExitCode exitCode) {
+ super(message);
+ this.exitCode = exitCode;
+ }
+
+ public ExitCausingRuntimeException(ExitCode exitCode) {
+ this.exitCode = exitCode;
+ }
+
+ public ExitCode getExitCode() {
+ return exitCode;
+ }
+ }
+
+ private static class HardwiredInfoItem implements BlazeModule.InfoItem {
+ private final InfoKey key;
+ private final BlazeRuntime runtime;
+ private final OptionsProvider commandOptions;
+
+ private HardwiredInfoItem(InfoKey key, BlazeRuntime runtime, OptionsProvider commandOptions) {
+ this.key = key;
+ this.runtime = runtime;
+ this.commandOptions = commandOptions;
+ }
+
+ @Override
+ public String getName() {
+ return key.getName();
+ }
+
+ @Override
+ public String getDescription() {
+ return key.getDescription();
+ }
+
+ @Override
+ public boolean isHidden() {
+ return key.isHidden();
+ }
+
+ @Override
+ public byte[] get(Supplier<BuildConfiguration> configurationSupplier) {
+ return print(getInfoItem(runtime, key, configurationSupplier, commandOptions));
+ }
+ }
+
+ private static class MakeInfoItem implements BlazeModule.InfoItem {
+ private final String name;
+ private final String value;
+
+ private MakeInfoItem(String name, String value) {
+ this.name = name;
+ this.value = value;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String getDescription() {
+ return "Make environment variable '" + name + "'";
+ }
+
+ @Override
+ public boolean isHidden() {
+ return false;
+ }
+
+ @Override
+ public byte[] get(Supplier<BuildConfiguration> configurationSupplier) {
+ return print(value);
+ }
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { }
+
+ @Override
+ public ExitCode exec(final BlazeRuntime runtime, final OptionsProvider optionsProvider) {
+ Options infoOptions = optionsProvider.getOptions(Options.class);
+
+ OutErr outErr = runtime.getReporter().getOutErr();
+ // Creating a BuildConfiguration is expensive and often unnecessary. Delay the creation until
+ // it is needed.
+ Supplier<BuildConfiguration> configurationSupplier = new Supplier<BuildConfiguration>() {
+ private BuildConfiguration configuration;
+ @Override
+ public BuildConfiguration get() {
+ if (configuration != null) {
+ return configuration;
+ }
+ try {
+ // In order to be able to answer configuration-specific queries, we need to setup the
+ // package path. Since info inherits all the build options, all the necessary information
+ // is available here.
+ runtime.setupPackageCache(
+ optionsProvider.getOptions(PackageCacheOptions.class),
+ runtime.getDefaultsPackageContent(optionsProvider));
+ // TODO(bazel-team): What if there are multiple configurations? [multi-config]
+ configuration = runtime
+ .getConfigurations(optionsProvider)
+ .getTargetConfigurations().get(0);
+ return configuration;
+ } catch (InvalidConfigurationException e) {
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ throw new ExitCausingRuntimeException(ExitCode.COMMAND_LINE_ERROR);
+ } catch (AbruptExitException e) {
+ throw new ExitCausingRuntimeException("unknown error: " + e.getMessage(),
+ e.getExitCode());
+ } catch (InterruptedException e) {
+ runtime.getReporter().handle(Event.error("interrupted"));
+ throw new ExitCausingRuntimeException(ExitCode.INTERRUPTED);
+ }
+ }
+ };
+
+ Map<String, BlazeModule.InfoItem> items = getInfoItemMap(runtime, optionsProvider);
+
+ try {
+ if (infoOptions.showMakeEnvironment) {
+ Map<String, String> makeEnv = configurationSupplier.get().getMakeEnvironment();
+ for (Map.Entry<String, String> entry : makeEnv.entrySet()) {
+ BlazeModule.InfoItem item = new MakeInfoItem(entry.getKey(), entry.getValue());
+ items.put(item.getName(), item);
+ }
+ }
+
+ List<String> residue = optionsProvider.getResidue();
+ if (residue.size() > 1) {
+ runtime.getReporter().handle(Event.error("at most one key may be specified"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ String key = residue.size() == 1 ? residue.get(0) : null;
+ if (key != null) { // print just the value for the specified key:
+ byte[] value;
+ if (items.containsKey(key)) {
+ value = items.get(key).get(configurationSupplier);
+ } else {
+ runtime.getReporter().handle(Event.error("unknown key: '" + key + "'"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ try {
+ outErr.getOutputStream().write(value);
+ outErr.getOutputStream().flush();
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error("Cannot write info block: " + e.getMessage()));
+ return ExitCode.ANALYSIS_FAILURE;
+ }
+ } else { // print them all
+ configurationSupplier.get(); // We'll need this later anyway
+ for (BlazeModule.InfoItem infoItem : items.values()) {
+ if (infoItem.isHidden()) {
+ continue;
+ }
+ outErr.getOutputStream().write(
+ (infoItem.getName() + ": ").getBytes(StandardCharsets.UTF_8));
+ outErr.getOutputStream().write(infoItem.get(configurationSupplier));
+ }
+ }
+ } catch (AbruptExitException e) {
+ return e.getExitCode();
+ } catch (ExitCausingRuntimeException e) {
+ return e.getExitCode();
+ } catch (IOException e) {
+ return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
+ }
+ return ExitCode.SUCCESS;
+ }
+
+ /**
+ * Compute and return the info for the given key. Only keys that are not hidden are supported
+ * here.
+ */
+ private static Object getInfoItem(BlazeRuntime runtime, InfoKey key,
+ Supplier<BuildConfiguration> configurationSupplier, OptionsProvider options) {
+ switch (key) {
+ // directories
+ case WORKSPACE : return runtime.getWorkspace();
+ case INSTALL_BASE : return runtime.getDirectories().getInstallBase();
+ case OUTPUT_BASE : return runtime.getOutputBase();
+ case EXECUTION_ROOT : return runtime.getExecRoot();
+ case OUTPUT_PATH : return runtime.getDirectories().getOutputPath();
+ // These are the only (non-hidden) info items that require a configuration, because the
+ // corresponding paths contain the short name. Maybe we should recommend using the symlinks
+ // or make them hidden by default?
+ case BLAZE_BIN : return configurationSupplier.get().getBinDirectory().getPath();
+ case BLAZE_GENFILES : return configurationSupplier.get().getGenfilesDirectory().getPath();
+ case BLAZE_TESTLOGS : return configurationSupplier.get().getTestLogsDirectory().getPath();
+
+ // logs
+ case COMMAND_LOG : return BlazeCommandDispatcher.getCommandLogPath(runtime.getOutputBase());
+ case MESSAGE_LOG :
+ // NB: Duplicated in EventLogModule
+ return runtime.getOutputBase().getRelative("message.log");
+
+ // misc
+ case RELEASE : return BlazeVersionInfo.instance().getReleaseName();
+ case SERVER_PID : return OsUtils.getpid();
+ case PACKAGE_PATH : return getPackagePath(options);
+
+ // memory statistics
+ case GC_COUNT :
+ case GC_TIME :
+ // The documentation is not very clear on what it means to have more than
+ // one GC MXBean, so we just sum them up.
+ int gcCount = 0;
+ int gcTime = 0;
+ for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) {
+ gcCount += gcBean.getCollectionCount();
+ gcTime += gcBean.getCollectionTime();
+ }
+ if (key == InfoKey.GC_COUNT) {
+ return gcCount + "";
+ } else {
+ return gcTime + "ms";
+ }
+
+ case MAX_HEAP_SIZE :
+ return StringUtilities.prettyPrintBytes(getMemoryUsage().getMax());
+ case USED_HEAP_SIZE :
+ case COMMITTED_HEAP_SIZE :
+ return StringUtilities.prettyPrintBytes(key == InfoKey.USED_HEAP_SIZE ?
+ getMemoryUsage().getUsed() : getMemoryUsage().getCommitted());
+
+ case USED_HEAP_SIZE_AFTER_GC :
+ // Note that this info value is not printed by default, but only when explicitly requested.
+ System.gc();
+ return StringUtilities.prettyPrintBytes(getMemoryUsage().getUsed());
+
+ case DEFAULTS_PACKAGE:
+ return runtime.getDefaultsPackageContent();
+
+ case BUILD_LANGUAGE:
+ return getBuildLanguageDefinition(runtime.getRuleClassProvider());
+
+ case DEFAULT_PACKAGE_PATH:
+ return Joiner.on(":").join(Constants.DEFAULT_PACKAGE_PATH);
+
+ default:
+ throw new IllegalArgumentException("missing implementation for " + key);
+ }
+ }
+
+ private static MemoryUsage getMemoryUsage() {
+ MemoryMXBean memBean = ManagementFactory.getMemoryMXBean();
+ return memBean.getHeapMemoryUsage();
+ }
+
+ /**
+ * Get the package_path variable for the given set of options.
+ */
+ private static String getPackagePath(OptionsProvider options) {
+ PackageCacheOptions packageCacheOptions =
+ options.getOptions(PackageCacheOptions.class);
+ return Joiner.on(":").join(packageCacheOptions.packagePath);
+ }
+
+ private static AllowedRuleClassInfo getAllowedRuleClasses(
+ Collection<RuleClass> ruleClasses, Attribute attr) {
+ AllowedRuleClassInfo.Builder info = AllowedRuleClassInfo.newBuilder();
+ info.setPolicy(AllowedRuleClassInfo.AllowedRuleClasses.ANY);
+
+ if (attr.isStrictLabelCheckingEnabled()) {
+ if (attr.getAllowedRuleClassesPredicate() != Predicates.<RuleClass>alwaysTrue()) {
+ info.setPolicy(AllowedRuleClassInfo.AllowedRuleClasses.SPECIFIED);
+ Predicate<RuleClass> filter = attr.getAllowedRuleClassesPredicate();
+ for (RuleClass otherClass : Iterables.filter(
+ ruleClasses, filter)) {
+ if (otherClass.isDocumented()) {
+ info.addAllowedRuleClass(otherClass.getName());
+ }
+ }
+ }
+ }
+
+ return info.build();
+ }
+
+ /**
+ * Returns a byte array containing a proto-buffer describing the build language.
+ */
+ private static byte[] getBuildLanguageDefinition(RuleClassProvider provider) {
+ BuildLanguage.Builder resultPb = BuildLanguage.newBuilder();
+ Collection<RuleClass> ruleClasses = provider.getRuleClassMap().values();
+ for (RuleClass ruleClass : ruleClasses) {
+ if (!ruleClass.isDocumented()) {
+ continue;
+ }
+
+ RuleDefinition.Builder rulePb = RuleDefinition.newBuilder();
+ rulePb.setName(ruleClass.getName());
+ for (Attribute attr : ruleClass.getAttributes()) {
+ if (!attr.isDocumented()) {
+ continue;
+ }
+
+ AttributeDefinition.Builder attrPb = AttributeDefinition.newBuilder();
+ attrPb.setName(attr.getName());
+ // The protocol compiler, in its infinite wisdom, generates the field as one of the
+ // integer type and the getTypeEnum() method is missing. WTF?
+ attrPb.setType(ProtoUtils.getDiscriminatorFromType(attr.getType()));
+ attrPb.setMandatory(attr.isMandatory());
+
+ if (Type.isLabelType(attr.getType())) {
+ attrPb.setAllowedRuleClasses(getAllowedRuleClasses(ruleClasses, attr));
+ }
+
+ rulePb.addAttribute(attrPb);
+ }
+
+ resultPb.addRule(rulePb);
+ }
+
+ return resultPb.build().toByteArray();
+ }
+
+ private static byte[] print(Object value) {
+ if (value instanceof byte[]) {
+ return (byte[]) value;
+ }
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ PrintWriter writer = new PrintWriter(outputStream);
+ writer.print(value.toString() + "\n");
+ writer.flush();
+ return outputStream.toByteArray();
+ }
+
+ static Map<String, BlazeModule.InfoItem> getInfoItemMap(
+ BlazeRuntime runtime, OptionsProvider commandOptions) {
+ Map<String, BlazeModule.InfoItem> result = new TreeMap<>(); // order by key
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ for (BlazeModule.InfoItem item : module.getInfoItems()) {
+ result.put(item.getName(), item);
+ }
+ }
+
+ for (InfoKey key : InfoKey.values()) {
+ BlazeModule.InfoItem item = new HardwiredInfoItem(key, runtime, commandOptions);
+ result.put(item.getName(), item);
+ }
+
+ return result;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java
new file mode 100644
index 0000000..d2e7bc0
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java
@@ -0,0 +1,90 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+
+/**
+ * An enumeration of all the valid info keys, excepting the make environment
+ * variables.
+ */
+public enum InfoKey {
+ // directories
+ WORKSPACE("workspace", "The working directory of the server."),
+ INSTALL_BASE("install_base", "The installation base directory."),
+ OUTPUT_BASE("output_base",
+ "A directory for shared Blaze state as well as tool and strategy specific subdirectories."),
+ EXECUTION_ROOT("execution_root",
+ "A directory that makes all input and output files visible to the build."),
+ OUTPUT_PATH("output_path", "Output directory"),
+ BLAZE_BIN("blaze-bin", "Configuration dependent directory for binaries."),
+ BLAZE_GENFILES("blaze-genfiles", "Configuration dependent directory for generated files."),
+ BLAZE_TESTLOGS("blaze-testlogs", "Configuration dependent directory for logs from a test run."),
+
+ // logs
+ COMMAND_LOG("command_log", "Location of the log containg the output from the build commands."),
+ MESSAGE_LOG("message_log" ,
+ "Location of a log containing machine readable message in LogMessage protobuf format."),
+
+ // misc
+ RELEASE("release", "Blaze release identifier"),
+ SERVER_PID("server_pid", "Blaze process id"),
+ PACKAGE_PATH("package_path", "The search path for resolving package labels."),
+
+ // memory statistics
+ USED_HEAP_SIZE("used-heap-size", "The amount of used memory in bytes. Note that this is not a "
+ + "good indicator of the actual memory use, as it includes any remaining inaccessible "
+ + "memory."),
+ USED_HEAP_SIZE_AFTER_GC("used-heap-size-after-gc",
+ "The amount of used memory in bytes after a call to System.gc().", true),
+ COMMITTED_HEAP_SIZE("committed-heap-size",
+ "The amount of memory in bytes that is committed for the Java virtual machine to use"),
+ MAX_HEAP_SIZE("max-heap-size",
+ "The maximum amount of memory in bytes that can be used for memory management."),
+ GC_COUNT("gc-count", "Number of garbage collection runs."),
+ GC_TIME("gc-time", "The approximate accumulated time spend on garbage collection."),
+
+ // These are deprecated, they still work, when explicitly requested, but are not shown by default
+
+ // These keys print multi-line messages and thus don't play well with grep. We don't print them
+ // unless explicitly requested
+ DEFAULTS_PACKAGE("defaults-package", "Default packages used as implicit dependencies", true),
+ BUILD_LANGUAGE("build-language", "A protobuffer with the build language structure", true),
+ DEFAULT_PACKAGE_PATH("default-package-path", "The default package path", true);
+
+ private final String name;
+ private final String description;
+ private final boolean hidden;
+
+ private InfoKey(String name, String description) {
+ this(name, description, false);
+ }
+
+ private InfoKey(String name, String description, boolean hidden) {
+ this.name = name;
+ this.description = description;
+ this.hidden = hidden;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public boolean isHidden() {
+ return hidden;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java
new file mode 100644
index 0000000..7b91dc7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java
@@ -0,0 +1,771 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.TreeMultimap;
+import com.google.devtools.build.lib.actions.MiddlemanAction;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.profiler.ProfileInfo;
+import com.google.devtools.build.lib.profiler.ProfileInfo.CriticalPathEntry;
+import com.google.devtools.build.lib.profiler.ProfileInfo.InfoListener;
+import com.google.devtools.build.lib.profiler.ProfilePhase;
+import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.profiler.chart.AggregatingChartCreator;
+import com.google.devtools.build.lib.profiler.chart.Chart;
+import com.google.devtools.build.lib.profiler.chart.ChartCreator;
+import com.google.devtools.build.lib.profiler.chart.DetailedChartCreator;
+import com.google.devtools.build.lib.profiler.chart.HtmlChartVisitor;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.StringUtil;
+import com.google.devtools.build.lib.util.TimeUtilities;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.common.options.Converters;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Command line wrapper for analyzing Blaze build profiles.
+ */
+@Command(name = "analyze-profile",
+ options = { ProfileCommand.ProfileOptions.class },
+ shortDescription = "Analyzes build profile data.",
+ help = "resource:analyze-profile.txt",
+ allowResidue = true,
+ mustRunInWorkspace = false)
+public final class ProfileCommand implements BlazeCommand {
+
+ private final String TWO_COLUMN_FORMAT = "%-37s %10s\n";
+ private final String THREE_COLUMN_FORMAT = "%-28s %10s %8s\n";
+
+ public static class DumpConverter extends Converters.StringSetConverter {
+ public DumpConverter() {
+ super("text", "raw", "text-unsorted", "raw-unsorted");
+ }
+ }
+
+ public static class ProfileOptions extends OptionsBase {
+ @Option(name = "dump",
+ abbrev='d',
+ converter = DumpConverter.class,
+ defaultValue = "null",
+ help = "output full profile data dump either in human-readable 'text' format or"
+ + " script-friendly 'raw' format, either sorted or unsorted.")
+ public String dumpMode;
+
+ @Option(name = "html",
+ defaultValue = "false",
+ help = "If present, an HTML file visualizing the tasks of the profiled build is created. "
+ + "The name of the html file is the name of the profile file plus '.html'.")
+ public boolean html;
+
+ @Option(name = "html_pixels_per_second",
+ defaultValue = "50",
+ help = "Defines the scale of the time axis of the task diagram. The unit is "
+ + "pixels per second. Default is 50 pixels per second. ")
+ public int htmlPixelsPerSecond;
+
+ @Option(name = "html_details",
+ defaultValue = "false",
+ help = "If --html_details is present, the task diagram contains all tasks of the profile. "
+ + "If --nohtml_details is present, an aggregated diagram is generated. The default is "
+ + "to generate an aggregated diagram.")
+ public boolean htmlDetails;
+
+ @Option(name = "vfs_stats",
+ defaultValue = "false",
+ help = "If present, include VFS path statistics.")
+ public boolean vfsStats;
+
+ @Option(name = "vfs_stats_limit",
+ defaultValue = "-1",
+ help = "Maximum number of VFS path statistics to print.")
+ public int vfsStatsLimit;
+ }
+
+ private Function<String, String> currentPathMapping = Functions.<String>identity();
+
+ private InfoListener getInfoListener(final BlazeRuntime runtime) {
+ return new InfoListener() {
+ private final EventHandler reporter = runtime.getReporter();
+
+ @Override
+ public void info(String text) {
+ reporter.handle(Event.info(text));
+ }
+
+ @Override
+ public void warn(String text) {
+ reporter.handle(Event.warn(text));
+ }
+ };
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+
+ @Override
+ public ExitCode exec(final BlazeRuntime runtime, OptionsProvider options) {
+ ProfileOptions opts =
+ options.getOptions(ProfileOptions.class);
+
+ if (!opts.vfsStats) {
+ opts.vfsStatsLimit = 0;
+ }
+
+ currentPathMapping = new Function<String, String>() {
+ @Override
+ public String apply(String input) {
+ if (runtime.getWorkspaceName().isEmpty()) {
+ return input;
+ } else {
+ return input.substring(input.lastIndexOf("/" + runtime.getWorkspaceName()) + 1);
+ }
+ }
+ };
+
+ PrintStream out = new PrintStream(runtime.getReporter().getOutErr().getOutputStream());
+ try {
+ runtime.getReporter().handle(Event.warn(
+ null, "This information is intended for consumption by Blaze developers"
+ + " only, and may change at any time. Script against it at your own risk"));
+
+ for (String name : options.getResidue()) {
+ Path profileFile = runtime.getWorkingDirectory().getRelative(name);
+ try {
+ ProfileInfo info = ProfileInfo.loadProfileVerbosely(
+ profileFile, getInfoListener(runtime));
+ if (opts.dumpMode != null) {
+ dumpProfile(runtime, info, out, opts.dumpMode);
+ } else if (opts.html) {
+ createHtml(runtime, info, profileFile, opts);
+ } else {
+ createText(runtime, info, out, opts);
+ }
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error(
+ null, "Failed to process file " + name + ": " + e.getMessage()));
+ }
+ }
+ } finally {
+ out.flush();
+ }
+ return ExitCode.SUCCESS;
+ }
+
+ private void createText(BlazeRuntime runtime, ProfileInfo info, PrintStream out,
+ ProfileOptions opts) {
+ List<ProfilePhaseStatistics> statistics = getStatistics(runtime, info, opts);
+
+ for (ProfilePhaseStatistics stat : statistics) {
+ String title = stat.getTitle();
+
+ if (!title.equals("")) {
+ out.println("\n=== " + title.toUpperCase() + " ===\n");
+ }
+ out.print(stat.getStatistics());
+ }
+ }
+
+ private void createHtml(BlazeRuntime runtime, ProfileInfo info, Path profileFile,
+ ProfileOptions opts)
+ throws IOException {
+ Path htmlFile =
+ profileFile.getParentDirectory().getChild(profileFile.getBaseName() + ".html");
+ List<ProfilePhaseStatistics> statistics = getStatistics(runtime, info, opts);
+
+ runtime.getReporter().handle(Event.info("Creating HTML output in " + htmlFile));
+
+ ChartCreator chartCreator =
+ opts.htmlDetails ? new DetailedChartCreator(info, statistics)
+ : new AggregatingChartCreator(info, statistics);
+ Chart chart = chartCreator.create();
+ OutputStream out = new BufferedOutputStream(htmlFile.getOutputStream());
+ try {
+ chart.accept(new HtmlChartVisitor(new PrintStream(out), opts.htmlPixelsPerSecond));
+ } finally {
+ try {
+ out.close();
+ } catch (IOException e) {
+ // Ignore
+ }
+ }
+ }
+
+ private List<ProfilePhaseStatistics> getStatistics(
+ BlazeRuntime runtime, ProfileInfo info, ProfileOptions opts) {
+ try {
+ ProfileInfo.aggregateProfile(info, getInfoListener(runtime));
+ runtime.getReporter().handle(Event.info("Analyzing relationships"));
+
+ info.analyzeRelationships();
+
+ List<ProfilePhaseStatistics> statistics = new ArrayList<>();
+
+ // Print phase durations and total execution time
+ ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(byteOutput, false, "UTF-8");
+ long duration = 0;
+ for (ProfilePhase phase : ProfilePhase.values()) {
+ ProfileInfo.Task phaseTask = info.getPhaseTask(phase);
+ if (phaseTask != null) {
+ duration += info.getPhaseDuration(phaseTask);
+ }
+ }
+ for (ProfilePhase phase : ProfilePhase.values()) {
+ ProfileInfo.Task phaseTask = info.getPhaseTask(phase);
+ if (phaseTask != null) {
+ long phaseDuration = info.getPhaseDuration(phaseTask);
+ out.printf(THREE_COLUMN_FORMAT, "Total " + phase.nick + " phase time",
+ TimeUtilities.prettyTime(phaseDuration), prettyPercentage(phaseDuration, duration));
+ }
+ }
+ out.printf(THREE_COLUMN_FORMAT, "Total run time", TimeUtilities.prettyTime(duration),
+ "100.00%");
+ statistics.add(new ProfilePhaseStatistics("Phase Summary Information",
+ new String(byteOutput.toByteArray(), "UTF-8")));
+
+ // Print details of major phases
+ if (duration > 0) {
+ statistics.add(formatInitPhaseStatistics(info, opts));
+ statistics.add(formatLoadingPhaseStatistics(info, opts));
+ statistics.add(formatAnalysisPhaseStatistics(info, opts));
+ ProfilePhaseStatistics stat = formatExecutionPhaseStatistics(info, opts);
+ if (stat != null) {
+ statistics.add(stat);
+ }
+ }
+
+ return statistics;
+ } catch (UnsupportedEncodingException e) {
+ throw new AssertionError("Should not happen since, UTF8 is available on all JVMs");
+ }
+ }
+
+ private void dumpProfile(
+ BlazeRuntime runtime, ProfileInfo info, PrintStream out, String dumpMode) {
+ if (!dumpMode.contains("unsorted")) {
+ ProfileInfo.aggregateProfile(info, getInfoListener(runtime));
+ }
+ if (dumpMode.contains("raw")) {
+ for (ProfileInfo.Task task : info.allTasksById) {
+ dumpRaw(task, out);
+ }
+ } else if (dumpMode.contains("unsorted")) {
+ for (ProfileInfo.Task task : info.allTasksById) {
+ dumpTask(task, out, 0);
+ }
+ } else {
+ for (ProfileInfo.Task task : info.rootTasksById) {
+ dumpTask(task, out, 0);
+ }
+ }
+ }
+
+ private void dumpTask(ProfileInfo.Task task, PrintStream out, int indent) {
+ StringBuilder builder = new StringBuilder(String.format(
+ "\n%s %s\nThread: %-6d Id: %-6d Parent: %d\nStart time: %-12s Duration: %s",
+ task.type, task.getDescription(), task.threadId, task.id, task.parentId,
+ TimeUtilities.prettyTime(task.startTime), TimeUtilities.prettyTime(task.duration)));
+ if (task.hasStats()) {
+ builder.append("\n");
+ ProfileInfo.AggregateAttr[] stats = task.getStatAttrArray();
+ for (ProfilerTask type : ProfilerTask.values()) {
+ ProfileInfo.AggregateAttr attr = stats[type.ordinal()];
+ if (attr != null) {
+ builder.append(type.toString().toLowerCase()).append("=(").
+ append(attr.count).append(", ").
+ append(TimeUtilities.prettyTime(attr.totalTime)).append(") ");
+ }
+ }
+ }
+ out.println(StringUtil.indent(builder.toString(), indent));
+ for (ProfileInfo.Task subtask : task.subtasks) {
+ dumpTask(subtask, out, indent + 1);
+ }
+ }
+
+ private void dumpRaw(ProfileInfo.Task task, PrintStream out) {
+ StringBuilder aggregateString = new StringBuilder();
+ ProfileInfo.AggregateAttr[] stats = task.getStatAttrArray();
+ for (ProfilerTask type : ProfilerTask.values()) {
+ ProfileInfo.AggregateAttr attr = stats[type.ordinal()];
+ if (attr != null) {
+ aggregateString.append(type.toString().toLowerCase()).append(",").
+ append(attr.count).append(",").append(attr.totalTime).append(" ");
+ }
+ }
+ out.println(
+ task.threadId + "|" + task.id + "|" + task.parentId + "|"
+ + task.startTime + "|" + task.duration + "|"
+ + aggregateString.toString().trim() + "|"
+ + task.type + "|" + task.getDescription());
+ }
+
+ /**
+ * Converts relative duration to the percentage string
+ * @return formatted percentage string or "N/A" if result is undefined.
+ */
+ private static String prettyPercentage(long duration, long total) {
+ if (total == 0) {
+ // Return "not available" string if total is 0 and result is undefined.
+ return "N/A";
+ }
+ return String.format("%5.2f%%", duration*100.0/total);
+ }
+
+ private void printCriticalPath(String title, PrintStream out, CriticalPathEntry path) {
+ out.println(String.format("\n%s (%s):", title,
+ TimeUtilities.prettyTime(path.cumulativeDuration)));
+
+ boolean lightCriticalPath = isLightCriticalPath(path);
+ out.println(lightCriticalPath ?
+ String.format("%6s %11s %8s %s", "Id", "Time", "Percentage", "Description")
+ : String.format("%6s %11s %8s %8s %s", "Id", "Time", "Share", "Critical", "Description"));
+
+ long totalPathTime = path.cumulativeDuration;
+ int middlemanCount = 0;
+ long middlemanDuration = 0L;
+ long middlemanCritTime = 0L;
+
+ for (; path != null ; path = path.next) {
+ if (path.task.id < 0) {
+ // Ignore fake actions.
+ continue;
+ } else if (path.task.getDescription().startsWith(MiddlemanAction.MIDDLEMAN_MNEMONIC + " ")
+ || path.task.getDescription().startsWith("TargetCompletionMiddleman")) {
+ // Aggregate middleman actions.
+ middlemanCount++;
+ middlemanDuration += path.duration;
+ middlemanCritTime += path.getCriticalTime();
+ } else {
+ String desc = path.task.getDescription().replace(':', ' ');
+ if (lightCriticalPath) {
+ out.println(String.format("%6d %11s %8s %s", path.task.id,
+ TimeUtilities.prettyTime(path.duration),
+ prettyPercentage(path.duration, totalPathTime),
+ desc));
+ } else {
+ out.println(String.format("%6d %11s %8s %8s %s", path.task.id,
+ TimeUtilities.prettyTime(path.duration),
+ prettyPercentage(path.duration, totalPathTime),
+ prettyPercentage(path.getCriticalTime(), totalPathTime), desc));
+ }
+ }
+ }
+ if (middlemanCount > 0) {
+ if (lightCriticalPath) {
+ out.println(String.format(" %11s %8s [%d middleman actions]",
+ TimeUtilities.prettyTime(middlemanDuration),
+ prettyPercentage(middlemanDuration, totalPathTime),
+ middlemanCount));
+ } else {
+ out.println(String.format(" %11s %8s %8s [%d middleman actions]",
+ TimeUtilities.prettyTime(middlemanDuration),
+ prettyPercentage(middlemanDuration, totalPathTime),
+ prettyPercentage(middlemanCritTime, totalPathTime), middlemanCount));
+ }
+ }
+ }
+
+ private boolean isLightCriticalPath(CriticalPathEntry path) {
+ return path.task.type == ProfilerTask.CRITICAL_PATH_COMPONENT;
+ }
+
+ private void printShortPhaseAnalysis(ProfileInfo info, PrintStream out, ProfilePhase phase) {
+ ProfileInfo.Task phaseTask = info.getPhaseTask(phase);
+ if (phaseTask != null) {
+ long phaseDuration = info.getPhaseDuration(phaseTask);
+ out.printf(TWO_COLUMN_FORMAT, "Total " + phase.nick + " phase time",
+ TimeUtilities.prettyTime(phaseDuration));
+ printTimeDistributionByType(info, out, phaseTask);
+ }
+ }
+
+ private void printTimeDistributionByType(ProfileInfo info, PrintStream out,
+ ProfileInfo.Task phaseTask) {
+ List<ProfileInfo.Task> taskList = info.getTasksForPhase(phaseTask);
+ long phaseDuration = info.getPhaseDuration(phaseTask);
+ long totalDuration = phaseDuration;
+ for (ProfileInfo.Task task : taskList) {
+ // Tasks on the phaseTask thread already accounted for in the phaseDuration.
+ if (task.threadId != phaseTask.threadId) {
+ totalDuration += task.duration;
+ }
+ }
+ boolean headerNeeded = true;
+ for (ProfilerTask type : ProfilerTask.values()) {
+ ProfileInfo.AggregateAttr stats = info.getStatsForType(type, taskList);
+ if (stats.count > 0 && stats.totalTime > 0) {
+ if (headerNeeded) {
+ out.println("\nTotal time (across all threads) spent on:");
+ out.println(String.format("%18s %8s %8s %11s", "Type", "Total", "Count", "Average"));
+ headerNeeded = false;
+ }
+ out.println(String.format("%18s %8s %8d %11s", type.toString(),
+ prettyPercentage(stats.totalTime, totalDuration), stats.count,
+ TimeUtilities.prettyTime(stats.totalTime / stats.count)));
+ }
+ }
+ }
+
+ static class Stat implements Comparable<Stat> {
+ public long duration;
+ public long frequency;
+
+ @Override
+ public int compareTo(Stat o) {
+ return this.duration == o.duration ? Long.compare(this.frequency, o.frequency)
+ : Long.compare(this.duration, o.duration);
+ }
+ }
+
+ /**
+ * Print the time spent on VFS operations on each path. Output is grouped by operation and sorted
+ * by descending duration. If multiple of the same VFS operation were logged for the same path,
+ * print the total duration.
+ *
+ * @param info profiling data.
+ * @param out output stream.
+ * @param phase build phase.
+ * @param limit maximum number of statistics to print, or -1 for no limit.
+ */
+ private void printVfsStatistics(ProfileInfo info, PrintStream out,
+ ProfilePhase phase, int limit) {
+ ProfileInfo.Task phaseTask = info.getPhaseTask(phase);
+ if (phaseTask == null) {
+ return;
+ }
+
+ if (limit == 0) {
+ return;
+ }
+
+ // Group into VFS operations and build maps from path to duration.
+
+ List<ProfileInfo.Task> taskList = info.getTasksForPhase(phaseTask);
+ EnumMap<ProfilerTask, Map<String, Stat>> stats = Maps.newEnumMap(ProfilerTask.class);
+
+ collectVfsEntries(stats, taskList);
+
+ if (!stats.isEmpty()) {
+ out.printf("\nVFS path statistics:\n");
+ out.printf("%15s %10s %10s %s\n", "Type", "Frequency", "Duration", "Path");
+ }
+
+ // Reverse the maps to get maps from duration to path. We use a TreeMultimap to sort by duration
+ // and because durations are not unique.
+
+ for (ProfilerTask type : stats.keySet()) {
+ Map<String, Stat> statsForType = stats.get(type);
+ TreeMultimap<Stat, String> sortedStats =
+ TreeMultimap.create(Ordering.natural().reverse(), Ordering.natural());
+
+ for (Map.Entry<String, Stat> stat : statsForType.entrySet()) {
+ sortedStats.put(stat.getValue(), stat.getKey());
+ }
+
+ int numPrinted = 0;
+ for (Map.Entry<Stat, String> stat : sortedStats.entries()) {
+ if (limit != -1 && numPrinted++ == limit) {
+ out.printf("... %d more ...\n", sortedStats.size() - limit);
+ break;
+ }
+ out.printf("%15s %10d %10s %s\n",
+ type.name(), stat.getKey().frequency, TimeUtilities.prettyTime(stat.getKey().duration),
+ stat.getValue());
+ }
+ }
+ }
+
+ private void collectVfsEntries(EnumMap<ProfilerTask, Map<String, Stat>> stats,
+ List<ProfileInfo.Task> taskList) {
+ for (ProfileInfo.Task task : taskList) {
+ collectVfsEntries(stats, Arrays.asList(task.subtasks));
+ if (!task.type.name().startsWith("VFS_")) {
+ continue;
+ }
+
+ Map<String, Stat> statsForType = stats.get(task.type);
+ if (statsForType == null) {
+ statsForType = Maps.newHashMap();
+ stats.put(task.type, statsForType);
+ }
+
+ String path = currentPathMapping.apply(task.getDescription());
+
+ Stat stat = statsForType.get(path);
+ if (stat == null) {
+ stat = new Stat();
+ }
+
+ stat.duration += task.duration;
+ stat.frequency++;
+ statsForType.put(path, stat);
+ }
+ }
+
+ /**
+ * Returns set of profiler tasks to be filtered from critical path.
+ * Also always filters out ACTION_LOCK and WAIT tasks to simulate
+ * unlimited resource critical path (see comments inside formatExecutionPhaseStatistics()
+ * method).
+ */
+ private EnumSet<ProfilerTask> getTypeFilter(ProfilerTask... tasks) {
+ EnumSet<ProfilerTask> filter = EnumSet.of(ProfilerTask.ACTION_LOCK, ProfilerTask.WAIT);
+ for (ProfilerTask task : tasks) {
+ filter.add(task);
+ }
+ return filter;
+ }
+
+ private ProfilePhaseStatistics formatInitPhaseStatistics(ProfileInfo info, ProfileOptions opts)
+ throws UnsupportedEncodingException {
+ return formatSimplePhaseStatistics(info, opts, "Init", ProfilePhase.INIT);
+ }
+
+ private ProfilePhaseStatistics formatLoadingPhaseStatistics(ProfileInfo info, ProfileOptions opts)
+ throws UnsupportedEncodingException {
+ return formatSimplePhaseStatistics(info, opts, "Loading", ProfilePhase.LOAD);
+ }
+
+ private ProfilePhaseStatistics formatAnalysisPhaseStatistics(ProfileInfo info,
+ ProfileOptions opts)
+ throws UnsupportedEncodingException {
+ return formatSimplePhaseStatistics(info, opts, "Analysis", ProfilePhase.ANALYZE);
+ }
+
+ private ProfilePhaseStatistics formatSimplePhaseStatistics(ProfileInfo info,
+ ProfileOptions opts,
+ String name,
+ ProfilePhase phase)
+ throws UnsupportedEncodingException {
+ ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(byteOutput, false, "UTF-8");
+
+ printShortPhaseAnalysis(info, out, phase);
+ printVfsStatistics(info, out, phase, opts.vfsStatsLimit);
+ return new ProfilePhaseStatistics(name + " Phase Information",
+ new String(byteOutput.toByteArray(), "UTF-8"));
+ }
+
+ private ProfilePhaseStatistics formatExecutionPhaseStatistics(ProfileInfo info,
+ ProfileOptions opts)
+ throws UnsupportedEncodingException {
+ ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(byteOutput, false, "UTF-8");
+
+ ProfileInfo.Task prepPhase = info.getPhaseTask(ProfilePhase.PREPARE);
+ ProfileInfo.Task execPhase = info.getPhaseTask(ProfilePhase.EXECUTE);
+ ProfileInfo.Task finishPhase = info.getPhaseTask(ProfilePhase.FINISH);
+ if (execPhase == null) {
+ return null;
+ }
+
+ List<ProfileInfo.Task> execTasks = info.getTasksForPhase(execPhase);
+ long graphTime = info.getStatsForType(ProfilerTask.ACTION_GRAPH, execTasks).totalTime;
+ long execTime = info.getPhaseDuration(execPhase) - graphTime;
+
+ if (prepPhase != null) {
+ out.printf(TWO_COLUMN_FORMAT, "Total preparation time",
+ TimeUtilities.prettyTime(info.getPhaseDuration(prepPhase)));
+ }
+ out.printf(TWO_COLUMN_FORMAT, "Total execution phase time",
+ TimeUtilities.prettyTime(info.getPhaseDuration(execPhase)));
+ if (finishPhase != null) {
+ out.printf(TWO_COLUMN_FORMAT, "Total time finalizing build",
+ TimeUtilities.prettyTime(info.getPhaseDuration(finishPhase)));
+ }
+ out.println("");
+ out.printf(TWO_COLUMN_FORMAT, "Action dependency map creation",
+ TimeUtilities.prettyTime(graphTime));
+ out.printf(TWO_COLUMN_FORMAT, "Actual execution time",
+ TimeUtilities.prettyTime(execTime));
+
+ EnumSet<ProfilerTask> typeFilter = EnumSet.noneOf(ProfilerTask.class);
+ CriticalPathEntry totalPath = info.getCriticalPath(typeFilter);
+ info.analyzeCriticalPath(typeFilter, totalPath);
+
+ typeFilter = getTypeFilter();
+ CriticalPathEntry optimalPath = info.getCriticalPath(typeFilter);
+ info.analyzeCriticalPath(typeFilter, optimalPath);
+
+ if (totalPath != null) {
+ printCriticalPathTimingBreakdown(info, totalPath, optimalPath, execTime, out);
+ } else {
+ out.println("\nCritical path not available because no action graph was generated.");
+ }
+
+ printTimeDistributionByType(info, out, execPhase);
+
+ if (totalPath != null) {
+ printCriticalPath("Critical path", out, totalPath);
+ // In light critical path we do not record scheduling delay data so it does not make sense
+ // to differentiate it.
+ if (!isLightCriticalPath(totalPath)) {
+ printCriticalPath("Critical path excluding scheduling delays", out, optimalPath);
+ }
+ }
+
+ if (info.getMissingActionsCount() > 0) {
+ out.println("\n" + info.getMissingActionsCount() + " action(s) are present in the"
+ + " action graph but missing instrumentation data. Most likely profile file"
+ + " has been created for the failed or aborted build.");
+ }
+
+ printVfsStatistics(info, out, ProfilePhase.EXECUTE, opts.vfsStatsLimit);
+
+ return new ProfilePhaseStatistics("Execution Phase Information",
+ new String(byteOutput.toByteArray(), "UTF-8"));
+ }
+
+ void printCriticalPathTimingBreakdown(ProfileInfo info, CriticalPathEntry totalPath,
+ CriticalPathEntry optimalPath, long execTime, PrintStream out) {
+ Preconditions.checkNotNull(totalPath);
+ Preconditions.checkNotNull(optimalPath);
+ // TODO(bazel-team): Print remote vs build stats recorded by CriticalPathStats
+ if (isLightCriticalPath(totalPath)) {
+ return;
+ }
+ out.println(totalPath.task.type);
+ // Worker thread pool scheduling delays for the actual critical path.
+ long workerWaitTime = 0;
+ long mainThreadWaitTime = 0;
+ for (ProfileInfo.CriticalPathEntry entry = totalPath; entry != null; entry = entry.next) {
+ workerWaitTime += info.getActionWaitTime(entry.task);
+ mainThreadWaitTime += info.getActionQueueTime(entry.task);
+ }
+ out.printf(TWO_COLUMN_FORMAT, "Worker thread scheduling delays",
+ TimeUtilities.prettyTime(workerWaitTime));
+ out.printf(TWO_COLUMN_FORMAT, "Main thread scheduling delays",
+ TimeUtilities.prettyTime(mainThreadWaitTime));
+
+ out.println("\nCritical path time:");
+ // Actual critical path.
+ long totalTime = totalPath.cumulativeDuration;
+ out.printf("%-37s %10s (%s of execution time)\n", "Actual time",
+ TimeUtilities.prettyTime(totalTime),
+ prettyPercentage(totalTime, execTime));
+ // Unlimited resource critical path. Essentially, we assume that if we
+ // remove all scheduling delays caused by resource semaphore contention,
+ // each action execution time would not change (even though load now would
+ // be substantially higher - so this assumption might be incorrect but it is
+ // still useful for modeling). Given those assumptions we calculate critical
+ // path excluding scheduling delays.
+ long optimalTime = optimalPath.cumulativeDuration;
+ out.printf("%-37s %10s (%s of execution time)\n", "Time excluding scheduling delays",
+ TimeUtilities.prettyTime(optimalTime),
+ prettyPercentage(optimalTime, execTime));
+
+ // Artificial critical path if we ignore all the time spent in all tasks,
+ // except time directly attributed to the ACTION tasks.
+ out.println("\nTime related to:");
+
+ EnumSet<ProfilerTask> typeFilter = EnumSet.allOf(ProfilerTask.class);
+ ProfileInfo.CriticalPathEntry path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "the builder overhead",
+ prettyPercentage(path.cumulativeDuration, totalTime));
+
+ typeFilter = getTypeFilter();
+ for (ProfilerTask task : ProfilerTask.values()) {
+ if (task.name().startsWith("VFS_")) {
+ typeFilter.add(task);
+ }
+ }
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "the VFS calls",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.ACTION_CHECK);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "the dependency checking",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.ACTION_EXECUTE);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "the execution setup",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.SPAWN, ProfilerTask.LOCAL_EXECUTION);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "local execution",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.SCANNER);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "the include scanner",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.REMOTE_EXECUTION, ProfilerTask.PROCESS_TIME,
+ ProfilerTask.LOCAL_PARSE, ProfilerTask.UPLOAD_TIME,
+ ProfilerTask.REMOTE_QUEUE, ProfilerTask.REMOTE_SETUP, ProfilerTask.FETCH);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, "Remote execution (cumulative)",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter( ProfilerTask.UPLOAD_TIME, ProfilerTask.REMOTE_SETUP);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " file uploads",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.FETCH);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " file fetching",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.PROCESS_TIME);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " process time",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.REMOTE_QUEUE);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " remote queueing",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.LOCAL_PARSE);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " remote execution parse",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+
+ typeFilter = getTypeFilter(ProfilerTask.REMOTE_EXECUTION);
+ path = info.getCriticalPath(typeFilter);
+ out.printf(TWO_COLUMN_FORMAT, " other remote activities",
+ prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java
new file mode 100644
index 0000000..2e5faf6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java
@@ -0,0 +1,93 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.CommonCommandOptions;
+import com.google.devtools.build.lib.runtime.ProjectFile;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.OptionPriority;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.List;
+
+/**
+ * Provides support for implementations for {@link BlazeCommand} to work with {@link ProjectFile}.
+ */
+public final class ProjectFileSupport {
+ static final String PROJECT_FILE_PREFIX = "+";
+
+ private ProjectFileSupport() {}
+
+ /**
+ * Reads any project files specified on the command line and updates the options parser
+ * accordingly. If project files cannot be read or if they contain unparsable options, or if they
+ * are not enabled, then it throws an exception instead.
+ */
+ public static void handleProjectFiles(BlazeRuntime runtime, OptionsParser optionsParser,
+ String command) throws AbruptExitException {
+ List<String> targets = optionsParser.getResidue();
+ ProjectFile.Provider projectFileProvider = runtime.getProjectFileProvider();
+ if (projectFileProvider != null && targets.size() > 0
+ && targets.get(0).startsWith(PROJECT_FILE_PREFIX)) {
+ if (targets.size() > 1) {
+ throw new AbruptExitException("Cannot handle more than one +<file> argument yet",
+ ExitCode.COMMAND_LINE_ERROR);
+ }
+ if (!optionsParser.getOptions(CommonCommandOptions.class).allowProjectFiles) {
+ throw new AbruptExitException("project file support is not enabled",
+ ExitCode.COMMAND_LINE_ERROR);
+ }
+ // TODO(bazel-team): This is currently treated as a path relative to the workspace - if the
+ // cwd is a subdirectory of the workspace, that will be surprising, and we should interpret it
+ // relative to the cwd instead.
+ PathFragment projectFilePath = new PathFragment(targets.get(0).substring(1));
+ List<Path> packagePath = PathPackageLocator.create(
+ optionsParser.getOptions(PackageCacheOptions.class).packagePath, runtime.getReporter(),
+ runtime.getWorkspace(), runtime.getWorkingDirectory()).getPathEntries();
+ ProjectFile projectFile = projectFileProvider.getProjectFile(packagePath, projectFilePath);
+ runtime.getReporter().handle(Event.info("Using " + projectFile.getName()));
+
+ try {
+ optionsParser.parse(
+ OptionPriority.RC_FILE, projectFile.getName(), projectFile.getCommandLineFor(command));
+ } catch (OptionsParsingException e) {
+ throw new AbruptExitException(e.getMessage(), ExitCode.COMMAND_LINE_ERROR);
+ }
+ }
+ }
+
+ /**
+ * Returns a list of targets from the options residue. If a project file is supplied as the first
+ * argument, it will be ignored, on the assumption that handleProjectFiles() has been called to
+ * process it.
+ */
+ public static List<String> getTargets(BlazeRuntime runtime, OptionsProvider options) {
+ List<String> targets = options.getResidue();
+ if (runtime.getProjectFileProvider() != null && targets.size() > 0
+ && targets.get(0).startsWith(PROJECT_FILE_PREFIX)) {
+ return targets.subList(1, targets.size());
+ }
+ return targets;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java
new file mode 100644
index 0000000..c5120cb
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java
@@ -0,0 +1,173 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
+import com.google.devtools.build.lib.query2.BlazeQueryEnvironment;
+import com.google.devtools.build.lib.query2.SkyframeQueryEnvironment;
+import com.google.devtools.build.lib.query2.engine.BlazeQueryEvalResult;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction;
+import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting;
+import com.google.devtools.build.lib.query2.engine.QueryException;
+import com.google.devtools.build.lib.query2.engine.QueryExpression;
+import com.google.devtools.build.lib.query2.output.OutputFormatter;
+import com.google.devtools.build.lib.query2.output.OutputFormatter.UnorderedFormatter;
+import com.google.devtools.build.lib.query2.output.QueryOptions;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.channels.ClosedByInterruptException;
+import java.util.Set;
+
+/**
+ * Command line wrapper for executing a query with blaze.
+ */
+@Command(name = "query",
+ options = { PackageCacheOptions.class,
+ QueryOptions.class },
+ help = "resource:query.txt",
+ shortDescription = "Executes a dependency graph query.",
+ allowResidue = true,
+ binaryStdOut = true,
+ canRunInOutputDirectory = true)
+public final class QueryCommand implements BlazeCommand {
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { }
+
+ /**
+ * Exit codes:
+ * 0 on successful evaluation.
+ * 1 if query evaluation did not complete.
+ * 2 if query parsing failed.
+ * 3 if errors were reported but evaluation produced a partial result
+ * (only when --keep_going is in effect.)
+ */
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ QueryOptions queryOptions = options.getOptions(QueryOptions.class);
+
+ try {
+ runtime.setupPackageCache(
+ options.getOptions(PackageCacheOptions.class),
+ runtime.getDefaultsPackageContent());
+ } catch (InterruptedException e) {
+ runtime.getReporter().handle(Event.error("query interrupted"));
+ return ExitCode.INTERRUPTED;
+ } catch (AbruptExitException e) {
+ runtime.getReporter().handle(Event.error(null, "Unknown error: " + e.getMessage()));
+ return e.getExitCode();
+ }
+
+ if (options.getResidue().isEmpty()) {
+ runtime.getReporter().handle(Event.error(
+ "missing query expression. Type 'blaze help query' for syntax and help"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ Iterable<OutputFormatter> formatters = runtime.getQueryOutputFormatters();
+ OutputFormatter formatter =
+ OutputFormatter.getFormatter(formatters, queryOptions.outputFormat);
+ if (formatter == null) {
+ runtime.getReporter().handle(Event.error(
+ String.format("Invalid output format '%s'. Valid values are: %s",
+ queryOptions.outputFormat, OutputFormatter.formatterNames(formatters))));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ String query = Joiner.on(' ').join(options.getResidue());
+
+ Set<Setting> settings = queryOptions.toSettings();
+ BlazeQueryEnvironment env = newQueryEnvironment(
+ runtime,
+ queryOptions.keepGoing,
+ queryOptions.loadingPhaseThreads,
+ settings);
+
+ // 1. Parse query:
+ QueryExpression expr;
+ try {
+ expr = QueryExpression.parse(query, env);
+ } catch (QueryException e) {
+ runtime.getReporter().handle(Event.error(
+ null, "Error while parsing '" + query + "': " + e.getMessage()));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ // 2. Evaluate expression:
+ BlazeQueryEvalResult<Target> result;
+ try {
+ result = env.evaluateQuery(expr);
+ } catch (QueryException e) {
+ // Keep consistent with reportBuildFileError()
+ runtime.getReporter().handle(Event.error(e.getMessage()));
+ return ExitCode.ANALYSIS_FAILURE;
+ }
+
+ // 3. Output results:
+ OutputFormatter.UnorderedFormatter unorderedFormatter = null;
+ if (!queryOptions.orderResults && formatter instanceof UnorderedFormatter) {
+ unorderedFormatter = (UnorderedFormatter) formatter;
+ }
+
+ PrintStream output = new PrintStream(runtime.getReporter().getOutErr().getOutputStream());
+ try {
+ if (unorderedFormatter != null) {
+ unorderedFormatter.outputUnordered(queryOptions, result.getResultSet(), output);
+ } else {
+ formatter.output(queryOptions, result.getResultGraph(), output);
+ }
+ } catch (ClosedByInterruptException e) {
+ runtime.getReporter().handle(Event.error("query interrupted"));
+ return ExitCode.INTERRUPTED;
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error("I/O error: " + e.getMessage()));
+ return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
+ } finally {
+ output.flush();
+ }
+ if (result.getResultSet().isEmpty()) {
+ runtime.getReporter().handle(Event.info("Empty results"));
+ }
+
+ return result.getSuccess() ? ExitCode.SUCCESS : ExitCode.PARTIAL_ANALYSIS_FAILURE;
+ }
+
+ @VisibleForTesting // for com.google.devtools.deps.gquery.test.QueryResultTestUtil
+ public static BlazeQueryEnvironment newQueryEnvironment(BlazeRuntime runtime,
+ boolean keepGoing, int loadingPhaseThreads, Set<Setting> settings) {
+ ImmutableList.Builder<QueryFunction> functions = ImmutableList.builder();
+ for (BlazeModule module : runtime.getBlazeModules()) {
+ functions.addAll(module.getQueryFunctions());
+ }
+ return new SkyframeQueryEnvironment(
+ runtime.getPackageManager().newTransitiveLoader(),
+ runtime.getPackageManager(),
+ runtime.getTargetPatternEvaluator(),
+ keepGoing, loadingPhaseThreads, runtime.getReporter(), settings, functions.build());
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java
new file mode 100644
index 0000000..b128d37
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java
@@ -0,0 +1,519 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.analysis.RunfilesSupport;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.analysis.config.RunUnder;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions;
+import com.google.devtools.build.lib.buildtool.BuildResult;
+import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils;
+import com.google.devtools.build.lib.buildtool.TargetValidator;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.exec.SymlinkTreeHelper;
+import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper;
+import com.google.devtools.build.lib.packages.OutputFile;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.packages.TargetUtils;
+import com.google.devtools.build.lib.packages.Type;
+import com.google.devtools.build.lib.pkgcache.LoadingFailedException;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.shell.AbnormalTerminationException;
+import com.google.devtools.build.lib.shell.BadExitStatusException;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.util.CommandBuilder;
+import com.google.devtools.build.lib.util.CommandDescriptionForm;
+import com.google.devtools.build.lib.util.CommandFailureUtils;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.FileType;
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.util.ShellEscaper;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Builds and run a target with the given command line arguments.
+ */
+@Command(name = "run",
+ builds = true,
+ options = { RunCommand.RunOptions.class },
+ inherits = { BuildCommand.class },
+ shortDescription = "Runs the specified target.",
+ help = "resource:run.txt",
+ allowResidue = true,
+ binaryStdOut = true,
+ binaryStdErr = true)
+public class RunCommand implements BlazeCommand {
+
+ public static class RunOptions extends OptionsBase {
+ @Option(name = "script_path",
+ category = "run",
+ defaultValue = "null",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "If set, write a shell script to the given file which invokes the "
+ + "target. If this option is set, the target is not run from Blaze. "
+ + "Use 'blaze run --script_path=foo //foo && foo' to invoke target '//foo' "
+ + "This differs from 'blaze run //foo' in that the Blaze lock is released "
+ + "and the executable is connected to the terminal's stdin.")
+ public PathFragment scriptPath;
+ }
+
+ @VisibleForTesting
+ public static final String SINGLE_TARGET_MESSAGE = "Blaze can only run a single target. "
+ + "Do not use wildcards that match more than one target";
+ @VisibleForTesting
+ public static final String NO_TARGET_MESSAGE = "No targets found to run";
+
+ private static final String PROCESS_WRAPPER = "process-wrapper";
+
+ // Value of --run_under as of the most recent command invocation.
+ private RunUnder currentRunUnder;
+
+ private static final FileType RUNFILES_MANIFEST = FileType.of(".runfiles_manifest");
+
+ @VisibleForTesting // productionVisibility = Visibility.PRIVATE
+ protected BuildResult processRequest(final BlazeRuntime runtime, BuildRequest request) {
+ return runtime.getBuildTool().processRequest(request, new TargetValidator() {
+ @Override
+ public void validateTargets(Collection<Target> targets, boolean keepGoing)
+ throws LoadingFailedException {
+ RunCommand.this.validateTargets(runtime.getReporter(), targets, keepGoing);
+ }
+ });
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { }
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ RunOptions runOptions = options.getOptions(RunOptions.class);
+ // This list should look like: ["//executable:target", "arg1", "arg2"]
+ List<String> targetAndArgs = options.getResidue();
+
+ // The user must at the least specify an executable target.
+ if (targetAndArgs.isEmpty()) {
+ runtime.getReporter().handle(Event.error("Must specify a target to run"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ String targetString = targetAndArgs.get(0);
+ List<String> runTargetArgs = targetAndArgs.subList(1, targetAndArgs.size());
+ RunUnder runUnder = options.getOptions(BuildConfiguration.Options.class).runUnder;
+
+ OutErr outErr = runtime.getReporter().getOutErr();
+ List<String> targets = (runUnder != null) && (runUnder.getLabel() != null)
+ ? ImmutableList.of(targetString, runUnder.getLabel().toString())
+ : ImmutableList.of(targetString);
+ BuildRequest request = BuildRequest.create(
+ this.getClass().getAnnotation(Command.class).name(), options,
+ runtime.getStartupOptionsProvider(), targets, outErr,
+ runtime.getCommandId(), runtime.getCommandStartTime());
+ if (request.getBuildOptions().compileOnly) {
+ String message = "The '" + getClass().getAnnotation(Command.class).name() +
+ "' command is incompatible with the --compile_only option";
+ runtime.getReporter().handle(Event.error(message));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ currentRunUnder = runUnder;
+ BuildResult result;
+ try {
+ result = processRequest(runtime, request);
+ } finally {
+ currentRunUnder = null;
+ }
+
+ if (!result.getSuccess()) {
+ runtime.getReporter().handle(Event.error("Build failed. Not running target"));
+ return result.getExitCondition();
+ }
+
+ // Make sure that we have exactly 1 built target (excluding --run_under),
+ // and that it is executable.
+ // These checks should only fail if keepGoing is true, because we already did
+ // validation before the build began. See {@link #validateTargets()}.
+ Collection<ConfiguredTarget> targetsBuilt = result.getSuccessfulTargets();
+ ConfiguredTarget targetToRun = null;
+ ConfiguredTarget runUnderTarget = null;
+
+ if (targetsBuilt != null) {
+ int maxTargets = runUnder != null && runUnder.getLabel() != null ? 2 : 1;
+ if (targetsBuilt.size() > maxTargets) {
+ runtime.getReporter().handle(Event.error(SINGLE_TARGET_MESSAGE));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ for (ConfiguredTarget target : targetsBuilt) {
+ ExitCode targetValidation = fullyValidateTarget(runtime, target);
+ if (targetValidation != ExitCode.SUCCESS) {
+ return targetValidation;
+ }
+ if (runUnder != null && target.getLabel().equals(runUnder.getLabel())) {
+ if (runUnderTarget != null) {
+ runtime.getReporter().handle(Event.error(
+ null, "Can't identify the run_under target from multiple options?"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ runUnderTarget = target;
+ } else if (targetToRun == null) {
+ targetToRun = target;
+ } else {
+ runtime.getReporter().handle(Event.error(SINGLE_TARGET_MESSAGE));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ }
+ }
+ // Handle target & run_under referring to the same target.
+ if ((targetToRun == null) && (runUnderTarget != null)) {
+ targetToRun = runUnderTarget;
+ }
+ if (targetToRun == null) {
+ runtime.getReporter().handle(Event.error(NO_TARGET_MESSAGE));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ Path executablePath = Preconditions.checkNotNull(
+ targetToRun.getProvider(FilesToRunProvider.class).getExecutable().getPath());
+ BuildConfiguration configuration = targetToRun.getConfiguration();
+ if (configuration == null) {
+ // The target may be an input file, which doesn't have a configuration. In that case, we
+ // choose any target configuration.
+ configuration = runtime.getBuildTool().getView().getConfigurationCollection()
+ .getTargetConfigurations().get(0);
+ }
+ Path workingDir;
+ try {
+ workingDir = ensureRunfilesBuilt(runtime, targetToRun);
+ } catch (CommandException e) {
+ runtime.getReporter().handle(Event.error("Error creating runfiles: " + e.getMessage()));
+ return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
+ }
+
+ List<String> args = runTargetArgs;
+
+ FilesToRunProvider provider = targetToRun.getProvider(FilesToRunProvider.class);
+ RunfilesSupport runfilesSupport = provider == null ? null : provider.getRunfilesSupport();
+ if (runfilesSupport != null && runfilesSupport.getArgs() != null) {
+ List<String> targetArgs = runfilesSupport.getArgs();
+ if (!targetArgs.isEmpty()) {
+ args = Lists.newArrayListWithCapacity(targetArgs.size() + runTargetArgs.size());
+ args.addAll(targetArgs);
+ args.addAll(runTargetArgs);
+ }
+ }
+
+ //
+ // We now have a unique executable ready to be run.
+ //
+ // We build up two different versions of the command to run: one with an absolute path, which
+ // we'll actually run, and a prettier one with the long absolute path to the executable
+ // replaced with a shorter relative path that uses the symlinks in the workspace.
+ PathFragment prettyExecutablePath =
+ OutputDirectoryLinksUtils.getPrettyPath(executablePath,
+ runtime.getWorkspaceName(), runtime.getWorkspace(),
+ options.getOptions(BuildRequestOptions.class).symlinkPrefix);
+ List<String> cmdLine = new ArrayList<>();
+ if (runOptions.scriptPath == null) {
+ cmdLine.add(runtime.getDirectories().getExecRoot()
+ .getRelative(runtime.getBinTools().getExecPath(PROCESS_WRAPPER)).getPathString());
+ cmdLine.add("-1");
+ cmdLine.add("15");
+ cmdLine.add("-");
+ cmdLine.add("-");
+ }
+ List<String> prettyCmdLine = new ArrayList<>();
+ // Insert the command prefix specified by the "--run_under=<command-prefix>" option
+ // at the start of the command line.
+ if (runUnder != null) {
+ String runUnderValue = runUnder.getValue();
+ if (runUnderTarget != null) {
+ // --run_under specifies a target. Get the corresponding executable.
+ // This must be an absolute path, because the run_under target is only
+ // in the runfiles of test targets.
+ runUnderValue = runUnderTarget
+ .getProvider(FilesToRunProvider.class).getExecutable().getPath().getPathString();
+ // If the run_under command contains any options, make sure to add them
+ // to the command line as well.
+ List<String> opts = runUnder.getOptions();
+ if (!opts.isEmpty()) {
+ runUnderValue += " " + ShellEscaper.escapeJoinAll(opts);
+ }
+ }
+ cmdLine.add(configuration.getShExecutable().getPathString());
+ cmdLine.add("-c");
+ cmdLine.add(runUnderValue + " " + executablePath.getPathString() + " " +
+ ShellEscaper.escapeJoinAll(args));
+ prettyCmdLine.add(configuration.getShExecutable().getPathString());
+ prettyCmdLine.add("-c");
+ prettyCmdLine.add(runUnderValue + " " + prettyExecutablePath.getPathString() + " " +
+ ShellEscaper.escapeJoinAll(args));
+ } else {
+ cmdLine.add(executablePath.getPathString());
+ cmdLine.addAll(args);
+ prettyCmdLine.add(prettyExecutablePath.getPathString());
+ prettyCmdLine.addAll(args);
+ }
+
+ // Add a newline between the blaze output and the binary's output.
+ outErr.printErrLn("");
+
+ if (runOptions.scriptPath != null) {
+ String unisolatedCommand = CommandFailureUtils.describeCommand(
+ CommandDescriptionForm.COMPLETE_UNISOLATED,
+ cmdLine, null, workingDir.getPathString());
+ if (writeScript(runtime, runOptions.scriptPath, unisolatedCommand)) {
+ return ExitCode.SUCCESS;
+ } else {
+ return ExitCode.RUN_FAILURE;
+ }
+ }
+
+ runtime.getReporter().handle(Event.info(
+ null, "Running command line: " + ShellEscaper.escapeJoinAll(prettyCmdLine)));
+
+ com.google.devtools.build.lib.shell.Command command = new CommandBuilder()
+ .addArgs(cmdLine).setEnv(runtime.getClientEnv()).setWorkingDir(workingDir).build();
+
+ try {
+ // The command API is a little strange in that the following statement
+ // will return normally only if the program exits with exit code 0.
+ // If it ends with any other code, we have to catch BadExitStatusException.
+ command.execute(com.google.devtools.build.lib.shell.Command.NO_INPUT,
+ com.google.devtools.build.lib.shell.Command.NO_OBSERVER,
+ outErr.getOutputStream(),
+ outErr.getErrorStream(),
+ true /* interruptible */).getTerminationStatus().getExitCode();
+ return ExitCode.SUCCESS;
+ } catch (BadExitStatusException e) {
+ String message = "Non-zero return code '"
+ + e.getResult().getTerminationStatus().getExitCode()
+ + "' from command: " + e.getMessage();
+ runtime.getReporter().handle(Event.error(message));
+ return ExitCode.RUN_FAILURE;
+ } catch (AbnormalTerminationException e) {
+ // The process was likely terminated by a signal in this case.
+ return ExitCode.INTERRUPTED;
+ } catch (CommandException e) {
+ runtime.getReporter().handle(Event.error("Error running program: " + e.getMessage()));
+ return ExitCode.RUN_FAILURE;
+ }
+ }
+
+ /**
+ * Ensures that runfiles are built for the specified target. If they already
+ * are, does nothing, otherwise builds them.
+ *
+ * @param target the target to build runfiles for.
+ * @return the path of the runfiles directory.
+ * @throws CommandException
+ */
+ private Path ensureRunfilesBuilt(BlazeRuntime runtime, ConfiguredTarget target)
+ throws CommandException {
+ FilesToRunProvider provider = target.getProvider(FilesToRunProvider.class);
+ RunfilesSupport runfilesSupport = provider == null ? null : provider.getRunfilesSupport();
+ if (runfilesSupport == null) {
+ return runtime.getWorkingDirectory();
+ }
+
+ Artifact manifest = runfilesSupport.getRunfilesManifest();
+ PathFragment runfilesDir = runfilesSupport.getRunfilesDirectoryExecPath();
+ Path workingDir = runtime.getExecRoot()
+ .getRelative(runfilesDir)
+ .getRelative(runtime.getRunfilesPrefix());
+
+ // When runfiles are not generated, getManifest() returns the
+ // .runfiles_manifest file, otherwise it returns the MANIFEST file. This is
+ // a handy way to check whether runfiles were built or not.
+ if (!RUNFILES_MANIFEST.matches(manifest.getFilename())) {
+ // Runfiles already built, nothing to do.
+ return workingDir;
+ }
+
+ SymlinkTreeHelper helper = new SymlinkTreeHelper(
+ manifest.getExecPath(),
+ runfilesDir,
+ false);
+ helper.createSymlinksUsingCommand(runtime.getExecRoot(), target.getConfiguration(),
+ runtime.getBinTools());
+ return workingDir;
+ }
+
+ private boolean writeScript(BlazeRuntime runtime, PathFragment scriptPathFrag, String cmd) {
+ final String SH_SHEBANG = "#!/bin/sh";
+ Path scriptPath = runtime.getWorkingDirectory().getRelative(scriptPathFrag);
+ try {
+ FileSystemUtils.writeContent(scriptPath, StandardCharsets.ISO_8859_1,
+ SH_SHEBANG + "\n" + cmd + " \"$@\"");
+ scriptPath.setExecutable(true);
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error("Error writing run script:" + e.getMessage()));
+ return false;
+ }
+ return true;
+ }
+
+ // Make sure we are building exactly 1 binary target.
+ // If keepGoing, we'll build all the targets even if they are non-binary.
+ private void validateTargets(Reporter reporter, Collection<Target> targets, boolean keepGoing)
+ throws LoadingFailedException {
+ Target targetToRun = null;
+ Target runUnderTarget = null;
+
+ boolean singleTargetWarningWasOutput = false;
+ int maxTargets = currentRunUnder != null && currentRunUnder.getLabel() != null ? 2 : 1;
+ if (targets.size() > maxTargets) {
+ warningOrException(reporter, SINGLE_TARGET_MESSAGE, keepGoing);
+ singleTargetWarningWasOutput = true;
+ }
+ for (Target target : targets) {
+ String targetError = validateTarget(target);
+ if (targetError != null) {
+ warningOrException(reporter, targetError, keepGoing);
+ }
+
+ if (currentRunUnder != null && target.getLabel().equals(currentRunUnder.getLabel())) {
+ // It's impossible to have two targets with the same label.
+ Preconditions.checkState(runUnderTarget == null);
+ runUnderTarget = target;
+ } else if (targetToRun == null) {
+ targetToRun = target;
+ } else {
+ if (!singleTargetWarningWasOutput) {
+ warningOrException(reporter, SINGLE_TARGET_MESSAGE, keepGoing);
+ }
+ return;
+ }
+ }
+ // Handle target & run_under referring to the same target.
+ if ((targetToRun == null) && (runUnderTarget != null)) {
+ targetToRun = runUnderTarget;
+ }
+ if (targetToRun == null) {
+ warningOrException(reporter, NO_TARGET_MESSAGE, keepGoing);
+ }
+ }
+
+ // If keepGoing, print a warning and return the given collection.
+ // Otherwise, throw InvalidTargetException.
+ private void warningOrException(Reporter reporter, String message,
+ boolean keepGoing) throws LoadingFailedException {
+ if (keepGoing) {
+ reporter.handle(Event.warn(message + ". Will continue anyway"));
+ } else {
+ throw new LoadingFailedException(message);
+ }
+ }
+
+ private static String notExecutableError(Target target) {
+ return "Cannot run target " + target.getLabel() + ": Not executable";
+ }
+
+ /** Returns null if the target is a runnable rule, or an appropriate error message otherwise. */
+ private static String validateTarget(Target target) {
+ return isExecutable(target)
+ ? null
+ : notExecutableError(target);
+ }
+
+ /**
+ * Performs all available validation checks on an individual target.
+ *
+ * @param target ConfiguredTarget to validate
+ * @return ExitCode.SUCCESS if all checks succeeded, otherwise a different error code.
+ */
+ private ExitCode fullyValidateTarget(BlazeRuntime runtime, ConfiguredTarget target) {
+ String targetError = validateTarget(target.getTarget());
+
+ if (targetError != null) {
+ runtime.getReporter().handle(Event.error(targetError));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ Artifact executable = target.getProvider(FilesToRunProvider.class).getExecutable();
+ if (executable == null) {
+ runtime.getReporter().handle(Event.error(notExecutableError(target.getTarget())));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+
+ // Shouldn't happen: We just validated the target.
+ Preconditions.checkState(executable != null,
+ "Could not find executable for target %s", target);
+ Path executablePath = executable.getPath();
+ try {
+ if (!executablePath.exists() || !executablePath.isExecutable()) {
+ runtime.getReporter().handle(Event.error(
+ null, "Non-existent or non-executable " + executablePath));
+ return ExitCode.BLAZE_INTERNAL_ERROR;
+ }
+ } catch (IOException e) {
+ runtime.getReporter().handle(Event.error(
+ "Error checking " + executablePath.getPathString() + ": " + e.getMessage()));
+ return ExitCode.LOCAL_ENVIRONMENTAL_ERROR;
+ }
+
+ return ExitCode.SUCCESS;
+ }
+
+ /**
+ * Return true iff {@code target} is a rule that has an executable file. This includes
+ * *_test rules, *_binary rules, and generated outputs.
+ */
+ private static boolean isExecutable(Target target) {
+ return isOutputFile(target) || isExecutableNonTestRule(target)
+ || TargetUtils.isTestRule(target);
+ }
+
+ /**
+ * Return true iff {@code target} is a rule that generates an executable file and is user-executed
+ * code.
+ */
+ private static boolean isExecutableNonTestRule(Target target) {
+ if (!(target instanceof Rule)) {
+ return false;
+ }
+ Rule rule = ((Rule) target);
+ if (rule.getRuleClassObject().hasAttr("$is_executable", Type.BOOLEAN)) {
+ return NonconfigurableAttributeMapper.of(rule).get("$is_executable", Type.BOOLEAN);
+ }
+ return false;
+ }
+
+ private static boolean isOutputFile(Target target) {
+ return (target instanceof OutputFile);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java
new file mode 100644
index 0000000..fb9ba39
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java
@@ -0,0 +1,71 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+/**
+ * The 'blaze shutdown' command.
+ */
+@Command(name = "shutdown",
+ options = { ShutdownCommand.Options.class },
+ allowResidue = false,
+ mustRunInWorkspace = false,
+ shortDescription = "Stops the Blaze server.",
+ help = "This command shuts down the memory resident Blaze server process.\n%{options}")
+public final class ShutdownCommand implements BlazeCommand {
+
+ public static class Options extends OptionsBase {
+
+ @Option(name="iff_heap_size_greater_than",
+ defaultValue = "0",
+ category = "misc",
+ help="Iff non-zero, then shutdown will only shut down the " +
+ "server if the total memory (in MB) consumed by the JVM " +
+ "exceeds this value.")
+ public int heapSizeLimit;
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options)
+ throws ShutdownBlazeServerException {
+
+ int limit = options.getOptions(Options.class).heapSizeLimit;
+
+ // Iff limit is non-zero, shut down the server if total memory exceeds the
+ // limit. totalMemory is the actual heap size that the VM currently uses
+ // *from the OS perspective*. That is, it's not the size occupied by all
+ // objects (which is totalMemory() - freeMemory()), and not the -Xmx
+ // (which is maxMemory()). It's really how much memory this process
+ // currently consumes, in addition to the JVM code and C heap.
+
+ if (limit == 0 ||
+ Runtime.getRuntime().totalMemory() > limit * 1000L * 1000) {
+ throw new ShutdownBlazeServerException(0);
+ }
+ return ExitCode.SUCCESS;
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java
new file mode 100644
index 0000000..70082ef
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java
@@ -0,0 +1,82 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.docgen.SkylarkDocumentationProcessor;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.Map;
+
+/**
+ * The 'doc_ext' command, which prints the extension API doc.
+ */
+@Command(name = "doc_ext",
+allowResidue = true,
+mustRunInWorkspace = false,
+shortDescription = "Prints help for commands, or the index.",
+help = "resource:skylark.txt")
+public final class SkylarkCommand implements BlazeCommand {
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options)
+ throws ShutdownBlazeServerException {
+ OutErr outErr = runtime.getReporter().getOutErr();
+ if (options.getResidue().isEmpty()) {
+ printTopLevelAPIDoc(outErr);
+ return ExitCode.SUCCESS;
+ }
+ if (options.getResidue().size() != 1) {
+ runtime.getReporter().handle(Event.error("Cannot specify more than one parameters"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ return printAPIDoc(options.getResidue().get(0), outErr, runtime.getReporter());
+ }
+
+ private ExitCode printAPIDoc(String param, OutErr outErr, Reporter reporter) {
+ String params[] = param.split("\\.");
+ if (params.length > 2) {
+ reporter.handle(Event.error("Identifier not found: " + param));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor();
+ String doc = processor.getCommandLineAPIDoc(params);
+ if (doc == null) {
+ reporter.handle(Event.error("Identifier not found: " + param));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ outErr.printOut(doc);
+ return ExitCode.SUCCESS;
+ }
+
+ private void printTopLevelAPIDoc(OutErr outErr) {
+ SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor();
+ outErr.printOut("Top level language modules, methods and objects:\n\n");
+ for (Map.Entry<String, String> entry : processor.collectTopLevelModules().entrySet()) {
+ outErr.printOut(entry.getKey() + ": " + entry.getValue());
+ }
+ }
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java
new file mode 100644
index 0000000..561c54a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java
@@ -0,0 +1,161 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.analysis.ConfiguredTarget;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.BuildResult;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
+import com.google.devtools.build.lib.rules.test.TestStrategy;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat;
+import com.google.devtools.build.lib.runtime.AggregatingTestListener;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeCommandEventHandler;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier;
+import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions;
+import com.google.devtools.build.lib.runtime.TestResultAnalyzer;
+import com.google.devtools.build.lib.runtime.TestResultNotifier;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter;
+import com.google.devtools.common.options.OptionPriority;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsParsingException;
+import com.google.devtools.common.options.OptionsProvider;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Handles the 'test' command on the Blaze command line.
+ */
+@Command(name = "test",
+ builds = true,
+ inherits = { BuildCommand.class },
+ options = { TestSummaryOptions.class },
+ shortDescription = "Builds and runs the specified test targets.",
+ help = "resource:test.txt",
+ allowResidue = true)
+public class TestCommand implements BlazeCommand {
+ private AnsiTerminalPrinter printer;
+
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser)
+ throws AbruptExitException {
+ ProjectFileSupport.handleProjectFiles(runtime, optionsParser, "test");
+
+ TestOutputFormat testOutput = optionsParser.getOptions(ExecutionOptions.class).testOutput;
+
+ if (testOutput == TestStrategy.TestOutputFormat.STREAMED) {
+ runtime.getReporter().handle(Event.warn(
+ "Streamed test output requested so all tests will be run locally, without sharding, " +
+ "one at a time"));
+ try {
+ optionsParser.parse(OptionPriority.SOFTWARE_REQUIREMENT,
+ "streamed output requires locally run tests, without sharding",
+ ImmutableList.of("--test_sharding_strategy=disabled", "--test_strategy=exclusive"));
+ } catch (OptionsParsingException e) {
+ throw new IllegalStateException("Known options failed to parse", e);
+ }
+ }
+ }
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ TestResultAnalyzer resultAnalyzer = new TestResultAnalyzer(
+ runtime.getExecRoot(),
+ options.getOptions(TestSummaryOptions.class),
+ options.getOptions(ExecutionOptions.class),
+ runtime.getEventBus());
+
+ printer = new AnsiTerminalPrinter(runtime.getReporter().getOutErr().getOutputStream(),
+ options.getOptions(BlazeCommandEventHandler.Options.class).useColor());
+
+ // Initialize test handler.
+ AggregatingTestListener testListener = new AggregatingTestListener(
+ resultAnalyzer, runtime.getEventBus(), runtime.getReporter());
+
+ runtime.getEventBus().register(testListener);
+ return doTest(runtime, options, testListener);
+ }
+
+ private ExitCode doTest(BlazeRuntime runtime,
+ OptionsProvider options,
+ AggregatingTestListener testListener) {
+ // Run simultaneous build and test.
+ List<String> targets = ProjectFileSupport.getTargets(runtime, options);
+ BuildRequest request = BuildRequest.create(
+ getClass().getAnnotation(Command.class).name(), options,
+ runtime.getStartupOptionsProvider(), targets,
+ runtime.getReporter().getOutErr(), runtime.getCommandId(), runtime.getCommandStartTime());
+ if (request.getBuildOptions().compileOnly) {
+ String message = "The '" + getClass().getAnnotation(Command.class).name() +
+ "' command is incompatible with the --compile_only option";
+ runtime.getReporter().handle(Event.error(message));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ request.setRunTests();
+
+ BuildResult buildResult = runtime.getBuildTool().processRequest(request, null);
+
+ Collection<ConfiguredTarget> testTargets = buildResult.getTestTargets();
+ // TODO(bazel-team): don't handle isEmpty here or fix up a bunch of tests
+ if (buildResult.getSuccessfulTargets() == null) {
+ // This can happen if there were errors in the target parsing or loading phase
+ // (original exitcode=BUILD_FAILURE) or if there weren't but --noanalyze was given
+ // (original exitcode=SUCCESS).
+ runtime.getReporter().handle(Event.error("Couldn't start the build. Unable to run tests"));
+ return buildResult.getSuccess() ? ExitCode.PARSING_FAILURE : buildResult.getExitCondition();
+ }
+ // TODO(bazel-team): the check above shadows NO_TESTS_FOUND, but switching the conditions breaks
+ // more tests
+ if (testTargets.isEmpty()) {
+ runtime.getReporter().handle(Event.error(
+ null, "No test targets were found, yet testing was requested"));
+ return buildResult.getSuccess() ? ExitCode.NO_TESTS_FOUND : buildResult.getExitCondition();
+ }
+
+ boolean buildSuccess = buildResult.getSuccess();
+ boolean testSuccess = analyzeTestResults(testTargets, testListener, options);
+
+ if (testSuccess && !buildSuccess) {
+ // If all tests run successfully, test summary should include warning if
+ // there were build errors not associated with the test targets.
+ printer.printLn(AnsiTerminalPrinter.Mode.ERROR
+ + "One or more non-test targets failed to build.\n"
+ + AnsiTerminalPrinter.Mode.DEFAULT);
+ }
+
+ return buildSuccess ?
+ (testSuccess ? ExitCode.SUCCESS : ExitCode.TESTS_FAILED)
+ : buildResult.getExitCondition();
+ }
+
+ /**
+ * Analyzes test results and prints summary information.
+ * Returns true if and only if all tests were successful.
+ */
+ private boolean analyzeTestResults(Collection<ConfiguredTarget> testTargets,
+ AggregatingTestListener listener,
+ OptionsProvider options) {
+ TestResultNotifier notifier = new TerminalTestResultNotifier(printer, options);
+ return listener.getAnalyzer().differentialAnalyzeAndReport(
+ testTargets, listener, notifier);
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java
new file mode 100644
index 0000000..0804cf6
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java
@@ -0,0 +1,49 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.runtime.commands;
+
+import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.runtime.BlazeCommand;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.common.options.OptionsParser;
+import com.google.devtools.common.options.OptionsProvider;
+
+/**
+ * The 'blaze version' command, which informs users about the blaze version
+ * information.
+ */
+@Command(name = "version",
+ options = {},
+ allowResidue = false,
+ mustRunInWorkspace = false,
+ help = "resource:version.txt",
+ shortDescription = "Prints version information for Blaze.")
+public final class VersionCommand implements BlazeCommand {
+ @Override
+ public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {}
+
+ @Override
+ public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) {
+ BlazeVersionInfo info = BlazeVersionInfo.instance();
+ if (info.getSummary() == null) {
+ runtime.getReporter().handle(Event.error("Version information not available"));
+ return ExitCode.COMMAND_LINE_ERROR;
+ }
+ runtime.getReporter().getOutErr().printOutLn(info.getSummary());
+ return ExitCode.SUCCESS;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt
new file mode 100644
index 0000000..0ef55a8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt
@@ -0,0 +1,14 @@
+
+Usage: blaze %{command} <options> <profile-files> [<profile-file> ...]
+
+Analyzes build profile data for the given profile data files.
+
+Analyzes each specified profile data file and prints the results. The
+input files must have been produced by the 'blaze build
+--profile=file' command.
+
+By default, a summary of the analysis is printed. For post-processing
+with scripts, the --dump=raw option is recommended, causing this
+command to dump profile data in easily-parsed format.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt
new file mode 100644
index 0000000..5e8d88a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt
@@ -0,0 +1,10 @@
+
+Usage: blaze %{command} <options> <targets>
+
+Builds the specified targets, using the options.
+
+See 'blaze help target-syntax' for details and examples on how to
+specify targets to build.
+
+%{options}
+
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt
new file mode 100644
index 0000000..11541ff
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt
@@ -0,0 +1,8 @@
+
+Usage: blaze canonicalize-flags <options> -- <options-to-canonicalize>
+
+Canonicalizes Blaze flags for the test and build commands. This command is
+intended to be used for tools that wish to check if two lists of options have
+the same effect at runtime.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt
new file mode 100644
index 0000000..7633888
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt
@@ -0,0 +1,10 @@
+
+Usage: blaze %{command} [<option> ...]
+
+Removes Blaze-created output, including all object files, and Blaze
+metadata.
+
+If '--expunge' is specified, the entire working tree will be removed
+and the server stopped.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt
new file mode 100644
index 0000000..a2040c8
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt
@@ -0,0 +1,7 @@
+
+Usage: blaze help [<command>]
+
+Prints a help page for the given command, or, if no command is
+specified, prints the index of available commands.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt
new file mode 100644
index 0000000..9c8b552
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt
@@ -0,0 +1,23 @@
+
+Usage: blaze info <options> [key]
+
+Displays information about the state of the blaze process in the
+form of several "key: value" pairs. This includes the locations of
+several output directories. Because some of the
+values are affected by the options passed to 'blaze build', the
+info command accepts the same set of options.
+
+A single non-option argument may be specified (e.g. "blaze-bin"), in
+which case only the value for that key will be printed.
+
+If --show_make_env is specified, the output includes the set of key/value
+pairs in the "Make" environment, accessible within BUILD files.
+
+The full list of keys and the meaning of their values is documented in
+the Blaze User Manual, and can be programmatically obtained with
+'blaze help info-keys'.
+
+See also 'blaze version' for more detailed blaze version
+information.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt
new file mode 100644
index 0000000..ce10211
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt
@@ -0,0 +1,19 @@
+
+Usage: blaze %{command} <options> <query-expression>
+
+Executes a query language expression over a specified subgraph of the
+build dependency graph.
+
+For example, to show all C++ test rules in the strings package, use:
+
+ % blaze query 'kind("cc_.*test", strings:*)'
+
+or to find all dependencies of chubby lockserver, use:
+
+ % blaze query 'deps(//path/to/package:target)'
+
+or to find a dependency path between //path/to/package:target and //dependency:
+
+ % blaze query 'somepath(//path/to/package:target, //dependency)'
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt
new file mode 100644
index 0000000..57283d5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt
@@ -0,0 +1,12 @@
+
+Usage: blaze %{command} <options> -- <binary target> <flags to binary>
+
+Build the specified target and run it with the given arguments.
+
+'run' accepts any 'build' options, and will inherit any defaults
+provided by .blazerc.
+
+If your script needs stdin or execution not constrained by the Blaze lock,
+use 'blaze run --script_path' to write a script and then execute it.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt
new file mode 100644
index 0000000..5414707
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt
@@ -0,0 +1,14 @@
+
+Startup options
+===============
+
+These options affect how Blaze starts up, or more specifically, how
+the virtual machine hosting Blaze starts up, and how the Blaze server
+starts up. These options must be specified to the left of the Blaze
+command (e.g. 'build'), and they must not contain any space between
+option name and value.
+
+Example:
+ % blaze --host_jvm_args=-Xmx1400m --output_base=/tmp/foo build //base
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt
new file mode 100644
index 0000000..1fac498
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt
@@ -0,0 +1,64 @@
+
+Target pattern syntax
+=====================
+
+The BUILD file label syntax is used to specify a single target. Target
+patterns generalize this syntax to sets of targets, and also support
+working-directory-relative forms, recursion, subtraction and filtering.
+Examples:
+
+Specifying a single target:
+
+ //foo/bar:wiz The single target '//foo/bar:wiz'.
+ foo/bar/wiz Equivalent to the first existing one of these:
+ //foo/bar:wiz
+ //foo:bar/wiz
+ //foo/bar Equivalent to '//foo/bar:bar'.
+
+Specifying all rules in a package:
+
+ //foo/bar:all Matches all rules in package 'foo/bar'.
+
+Specifying all rules recursively beneath a package:
+
+ //foo/...:all Matches all rules in all packages beneath directory 'foo'.
+ //foo/... (ditto)
+
+Working-directory relative forms: (assume cwd = 'workspace/foo')
+
+ Target patterns which do not begin with '//' are taken relative to
+ the working directory. Patterns which begin with '//' are always
+ absolute.
+
+ ...:all Equivalent to '//foo/...:all'.
+ ... (ditto)
+
+ bar/...:all Equivalent to '//foo/bar/...:all'.
+ bar/... (ditto)
+
+ bar:wiz Equivalent to '//foo/bar:wiz'.
+ :foo Equivalent to '//foo:foo'.
+
+ bar:all Equivalent to '//foo/bar:all'.
+ :all Equivalent to '//foo:all'.
+
+Summary of target wildcards:
+
+ :all, Match all rules in the specified packages.
+ :*, :all-targets Match all targets (rules and files) in the specified
+ packages, including .par and _deploy.jar files.
+
+Subtractive patterns:
+
+ Target patterns may be preceded by '-', meaning they should be
+ subtracted from the set of targets accumulated by preceding
+ patterns. For example:
+
+ % blaze build -- foo/... -foo/contrib/...
+
+ builds everything in 'foo', except 'contrib'. In case a target not
+ under 'contrib' depends on something under 'contrib' though, in order to
+ build the former blaze has to build the latter too. As usual, the '--' is
+ required to prevent '-b' from being interpreted as an option.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt
new file mode 100644
index 0000000..a1f0523
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt
@@ -0,0 +1,15 @@
+
+Usage: blaze %{command} <options> <test-targets>
+
+Builds the specified targets and runs all test targets among them (test targets
+might also need to satisfy provided tag, size or language filters) using
+the specified options.
+
+This command accepts all valid options to 'build', and inherits
+defaults for 'build' from your .blazerc. If you don't use .blazerc,
+don't forget to pass all your 'build' options to '%{command}' too.
+
+See 'blaze help target-syntax' for details and examples on how to
+specify targets.
+
+%{options}
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt
new file mode 100644
index 0000000..10e1df7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt
@@ -0,0 +1,3 @@
+Prints the version information that was embedded when blaze was built.
+
+%{options}
