Open source DynamicExecutionModule
RELNOTES: Dynamic execution is now available with --experimental_spawn_strategy. Dynamic execution allows a build action to run locally and remotely simultaneously, and Bazel picks the fastest action. This provides the best of both worlds: faster clean builds than pure local builds, and faster incremental builds than pure remote builds.
PiperOrigin-RevId: 222446721
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index 0d05429..af2a42b 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -30,6 +30,7 @@
"//src/main/java/com/google/devtools/build/lib/collect/nestedset:srcs",
"//src/main/java/com/google/devtools/build/lib/collect:srcs",
"//src/main/java/com/google/devtools/build/lib/concurrent:srcs",
+ "//src/main/java/com/google/devtools/build/lib/dynamic:srcs",
"//src/main/java/com/google/devtools/build/lib/exec/apple:srcs",
"//src/main/java/com/google/devtools/build/lib/exec/local:srcs",
"//src/main/java/com/google/devtools/build/lib/graph:srcs",
@@ -766,6 +767,7 @@
":build-base",
"//src/main/java/com/google/devtools/build/lib/bazel/debug:workspace-rule-module",
"//src/main/java/com/google/devtools/build/lib/buildeventservice",
+ "//src/main/java/com/google/devtools/build/lib/dynamic",
"//src/main/java/com/google/devtools/build/lib/metrics:metrics_module",
"//src/main/java/com/google/devtools/build/lib/profiler/callcounts:callcounts_module",
"//src/main/java/com/google/devtools/build/lib/profiler/memory:allocationtracker_module",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
index 029c1ac..ddcc464 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
@@ -55,6 +55,7 @@
com.google.devtools.build.lib.standalone.StandaloneModule.class,
com.google.devtools.build.lib.sandbox.SandboxModule.class,
com.google.devtools.build.lib.runtime.BuildSummaryStatsModule.class,
+ com.google.devtools.build.lib.dynamic.DynamicExecutionModule.class,
com.google.devtools.build.lib.bazel.rules.BazelRulesModule.class,
com.google.devtools.build.lib.bazel.rules.BazelStrategyModule.class,
com.google.devtools.build.lib.buildeventservice.BazelBuildEventServiceModule.class,
diff --git a/src/main/java/com/google/devtools/build/lib/dynamic/BUILD b/src/main/java/com/google/devtools/build/lib/dynamic/BUILD
new file mode 100644
index 0000000..57e542d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/dynamic/BUILD
@@ -0,0 +1,25 @@
+package(default_visibility = ["//src:__subpackages__"])
+
+filegroup(
+ name = "srcs",
+ srcs = glob(["**"]),
+ visibility = ["//src/main/java/com/google/devtools/build/lib:__pkg__"],
+)
+
+java_library(
+ name = "dynamic",
+ srcs = glob(["*.java"]),
+ deps = [
+ "//src/main/java/com/google/devtools/build/lib:build-base",
+ "//src/main/java/com/google/devtools/build/lib:events",
+ "//src/main/java/com/google/devtools/build/lib:io",
+ "//src/main/java/com/google/devtools/build/lib:runtime",
+ "//src/main/java/com/google/devtools/build/lib/actions",
+ "//src/main/java/com/google/devtools/build/lib/concurrent",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
+ "//src/main/java/com/google/devtools/common/options",
+ "//third_party:auto_value",
+ "//third_party:guava",
+ "//third_party:jsr305",
+ ],
+)
diff --git a/src/main/java/com/google/devtools/build/lib/dynamic/DynamicExecutionModule.java b/src/main/java/com/google/devtools/build/lib/dynamic/DynamicExecutionModule.java
new file mode 100644
index 0000000..750e2b4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/dynamic/DynamicExecutionModule.java
@@ -0,0 +1,112 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.dynamic;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.ExecutorInitException;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.actions.Spawns;
+import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.concurrent.ExecutorUtil;
+import com.google.devtools.build.lib.exec.ExecutionPolicy;
+import com.google.devtools.build.lib.exec.ExecutorBuilder;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.common.options.OptionsBase;
+import java.util.Arrays;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * {@link BlazeModule} providing support for dynamic spawn execution and scheduling.
+ */
+public class DynamicExecutionModule extends BlazeModule {
+ private ExecutorService executorService;
+
+ @Override
+ public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
+ return "build".equals(command.name())
+ ? ImmutableList.<Class<? extends OptionsBase>>of(DynamicExecutionOptions.class)
+ : ImmutableList.<Class<? extends OptionsBase>>of();
+ }
+
+ @Override
+ public void beforeCommand(CommandEnvironment env) {
+ executorService =
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("dynamic-execution-thread-%d").build());
+ env.getEventBus().register(this);
+ }
+
+ /**
+ * Adds a strategy that backs the dynamic scheduler to the executor builder.
+ *
+ * @param builder the executor builder to modify
+ * @param name the name of the strategy
+ * @param flagName name of the flag the strategy came from; used for error reporting
+ * purposes only
+ * @throws ExecutorInitException if the provided strategy would cause a scheduling cycle
+ */
+ private static void addBackingStrategy(ExecutorBuilder builder, String name, String flagName)
+ throws ExecutorInitException {
+ ExecutionStrategy strategy = DynamicSpawnStrategy.class.getAnnotation(ExecutionStrategy.class);
+ checkNotNull(strategy, "DynamicSpawnStrategy lacks expected ExecutionStrategy annotation");
+
+ if (Arrays.asList(strategy.name()).contains(name)) {
+ throw new ExecutorInitException("Cannot use strategy " + name + " in flag " + flagName
+ + " as it would create a cycle during execution");
+ }
+
+ builder.addStrategyByContext(SpawnActionContext.class, name);
+ }
+
+ @Override
+ public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorBuilder builder)
+ throws ExecutorInitException {
+ DynamicExecutionOptions options = env.getOptions().getOptions(DynamicExecutionOptions.class);
+ if (options.internalSpawnScheduler) {
+ builder.addActionContext(
+ new DynamicSpawnStrategy(executorService, options, this::getExecutionPolicy));
+ builder.addStrategyByContext(SpawnActionContext.class, "dynamic");
+ addBackingStrategy(builder, options.dynamicLocalStrategy, "--dynamic_local_strategy");
+ addBackingStrategy(builder, options.dynamicRemoteStrategy, "--dynamic_remote_strategy");
+ addBackingStrategy(builder, options.dynamicWorkerStrategy, "--dynamic_worker_strategy");
+ }
+ }
+
+ /**
+ * Use the {@link Spawn} metadata to determine if it can be executed locally, remotely, or both.
+ * @param spawn the {@link Spawn} action
+ * @return the {@link ExecutionPolicy} containing local/remote execution policies
+ */
+ protected ExecutionPolicy getExecutionPolicy(Spawn spawn) {
+ if (!Spawns.mayBeExecutedRemotely(spawn)) {
+ return ExecutionPolicy.LOCAL_EXECUTION_ONLY;
+ }
+
+ return ExecutionPolicy.ANYWHERE;
+ }
+
+ @Override
+ public void afterCommand() {
+ ExecutorUtil.interruptibleShutdown(executorService);
+ executorService = null;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/dynamic/DynamicExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/dynamic/DynamicExecutionOptions.java
new file mode 100644
index 0000000..8b13c0c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/dynamic/DynamicExecutionOptions.java
@@ -0,0 +1,94 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.dynamic;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionDocumentationCategory;
+import com.google.devtools.common.options.OptionEffectTag;
+import com.google.devtools.common.options.OptionsBase;
+
+/**
+ * Options related to dynamic spawn execution.
+ */
+public class DynamicExecutionOptions extends OptionsBase {
+
+ @Option(
+ name = "experimental_spawn_scheduler",
+ documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ defaultValue = "null",
+ help =
+ "Run actions locally instead of remotely for incremental builds as long as enough "
+ + "resources are available to execute all runnable actions in parallel.",
+ expansion = {"--internal_spawn_scheduler", "--spawn_strategy=dynamic"})
+ public Void experimentalSpawnScheduler;
+
+ @Option(
+ name = "internal_spawn_scheduler",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ defaultValue = "false",
+ help =
+ "Placeholder option so that we can tell in Blaze whether the spawn scheduler was "
+ + "enabled."
+ )
+ public boolean internalSpawnScheduler;
+
+ @Option(
+ name = "dynamic_local_strategy",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ defaultValue = "sandboxed",
+ help = "Strategy to use when the dynamic spawn scheduler decides to run an action locally."
+ )
+ public String dynamicLocalStrategy;
+
+ @Option(
+ name = "dynamic_remote_strategy",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ defaultValue = "remote",
+ help = "Strategy to use when the dynamic spawn scheduler decides to run an action remotely."
+ )
+ public String dynamicRemoteStrategy;
+
+ @Option(
+ name = "dynamic_worker_strategy",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ defaultValue = "worker",
+ help = "Strategy to use when the dynamic spawn scheduler decides to run an action in a"
+ + " worker."
+ )
+ public String dynamicWorkerStrategy;
+
+ @Option(
+ name = "experimental_local_execution_delay",
+ documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ defaultValue = "1000",
+ help =
+ "How many milliseconds should local execution be delayed, if remote execution was faster"
+ + " during a build at least once?"
+ )
+ public int localExecutionDelay;
+
+ @Option(
+ name = "experimental_debug_spawn_scheduler",
+ documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+ effectTags = {OptionEffectTag.UNKNOWN},
+ defaultValue = "false"
+ )
+ public boolean debugSpawnScheduler;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/dynamic/DynamicSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/dynamic/DynamicSpawnStrategy.java
new file mode 100644
index 0000000..d37d16c
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/dynamic/DynamicSpawnStrategy.java
@@ -0,0 +1,414 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.dynamic;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.io.Files;
+import com.google.devtools.build.lib.actions.ActionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionRequirements;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.ExecutorInitException;
+import com.google.devtools.build.lib.actions.SandboxedSpawnActionContext;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.actions.SpawnResult;
+import com.google.devtools.build.lib.actions.UserExecException;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.exec.ExecutionPolicy;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+
+/**
+ * A spawn strategy that speeds up incremental builds while not slowing down full builds.
+ *
+ * <p>It tries to run spawn actions on the local machine, which is faster than running them
+ * remotely, unless there are not enough resources available to run all of them in parallel, in
+ * which case the increased parallelism of remote execution is likely to provide higher overall
+ * throughput.
+ *
+ * <p>The strategy will delegate execution to either a remote spawn strategy for remote execution or
+ * the sandboxed strategy for local execution. The general idea is to run spawns on the local
+ * machine, as long as it can keep up with the required parallelism. If it fails to do so, we assume
+ * that the build would benefit from the increased parallelism available when using remote
+ * execution, so we switch to using remote execution for all following spawns.
+ *
+ * <p>One might ask, why we don't run spawns on the workstation all the time and just "spill over"
+ * actions to remote execution when there are no local resources available. This would work, except
+ * that the cost of transferring action inputs and outputs from the local machine to and from remote
+ * executors over the network is way too high - there is no point in executing an action locally and
+ * save 0.5s of time, when it then takes us 5 seconds to upload the results to remote executors for
+ * another action that's scheduled to run there.
+ */
+@ExecutionStrategy(
+ name = {"dynamic", "dynamic_worker"},
+ contextType = SpawnActionContext.class
+)
+public class DynamicSpawnStrategy implements SpawnActionContext {
+ private static final Logger logger = Logger.getLogger(DynamicSpawnStrategy.class.getName());
+
+ enum StrategyIdentifier {
+ NONE("unknown"),
+ LOCAL("locally"),
+ REMOTE("remotely");
+
+ private final String prettyName;
+
+ StrategyIdentifier(String prettyName) {
+ this.prettyName = prettyName;
+ }
+
+ String prettyName() {
+ return prettyName;
+ }
+ }
+
+ @AutoValue
+ abstract static class DynamicExecutionResult {
+ static DynamicExecutionResult create(
+ StrategyIdentifier strategyIdentifier,
+ @Nullable FileOutErr fileOutErr,
+ @Nullable ExecException execException,
+ List<SpawnResult> spawnResults) {
+ return new AutoValue_DynamicSpawnStrategy_DynamicExecutionResult(
+ strategyIdentifier, fileOutErr, execException, spawnResults);
+ }
+
+ abstract StrategyIdentifier strategyIdentifier();
+
+ @Nullable
+ abstract FileOutErr fileOutErr();
+
+ @Nullable
+ abstract ExecException execException();
+
+ /**
+ * Returns a list of SpawnResults associated with executing a Spawn.
+ *
+ * <p>The list will typically contain one element, but could contain zero elements if spawn
+ * execution did not complete, or multiple elements if multiple sub-spawns were executed.
+ */
+ abstract List<SpawnResult> spawnResults();
+ }
+
+ private static final ImmutableSet<String> WORKER_BLACKLISTED_MNEMONICS =
+ ImmutableSet.of("JavaDeployJar");
+
+ private final ExecutorService executorService;
+ private final DynamicExecutionOptions options;
+ private final Function<Spawn, ExecutionPolicy> getExecutionPolicy;
+ private final AtomicBoolean delayLocalExecution = new AtomicBoolean(false);
+
+ private @Nullable SandboxedSpawnActionContext remoteStrategy;
+ private @Nullable SandboxedSpawnActionContext localStrategy;
+ private @Nullable SandboxedSpawnActionContext workerStrategy;
+
+ /**
+ * Constructs a {@code DynamicSpawnStrategy}.
+ *
+ * @param executorService an {@link ExecutorService} that will be used to run Spawn actions.
+ */
+ public DynamicSpawnStrategy(
+ ExecutorService executorService,
+ DynamicExecutionOptions options,
+ Function<Spawn, ExecutionPolicy> getExecutionPolicy) {
+ this.executorService = executorService;
+ this.options = options;
+ this.getExecutionPolicy = getExecutionPolicy;
+ }
+
+ /**
+ * Searches for a sandboxed spawn strategy with the given name.
+ *
+ * @param usedContexts the action contexts used during the build
+ * @param name the name of the spawn strategy we are interested in
+ * @return a sandboxed spawn strategy
+ * @throws ExecutorInitException if the spawn strategy does not exist, or if it exists but is not
+ * sandboxed
+ */
+ private SandboxedSpawnActionContext findStrategy(
+ Iterable<ActionContext> usedContexts, String name) throws ExecutorInitException {
+ for (ActionContext context : usedContexts) {
+ ExecutionStrategy strategy = context.getClass().getAnnotation(ExecutionStrategy.class);
+ if (strategy != null && Arrays.asList(strategy.name()).contains(name)) {
+ if (!(context instanceof SandboxedSpawnActionContext)) {
+ throw new ExecutorInitException("Requested strategy " + name + " exists but does not "
+ + "support sandboxing");
+ }
+ return (SandboxedSpawnActionContext) context;
+ }
+ }
+ throw new ExecutorInitException("Requested strategy " + name + " does not exist");
+ }
+
+ @Override
+ public void executorCreated(Iterable<ActionContext> usedContexts) throws ExecutorInitException {
+ localStrategy = findStrategy(usedContexts, options.dynamicLocalStrategy);
+ remoteStrategy = findStrategy(usedContexts, options.dynamicRemoteStrategy);
+ workerStrategy = findStrategy(usedContexts, options.dynamicWorkerStrategy);
+ }
+
+ @Override
+ public List<SpawnResult> exec(
+ final Spawn spawn, final ActionExecutionContext actionExecutionContext)
+ throws ExecException, InterruptedException {
+
+ ExecutionPolicy executionPolicy = getExecutionPolicy.apply(spawn);
+
+ // If a Spawn cannot run remotely, we must always execute it locally. Resources will already
+ // have been acquired by Skyframe for us.
+ if (executionPolicy.canRunLocallyOnly()) {
+ return runLocally(spawn, actionExecutionContext, null);
+ }
+
+ // If a Spawn cannot run locally, we must always execute it remotely. For remote execution,
+ // local resources should not be acquired.
+ if (executionPolicy.canRunRemotelyOnly()) {
+ return runRemotely(spawn, actionExecutionContext, null);
+ }
+
+ // At this point we have a Spawn that can run locally and can run remotely. Run it in parallel
+ // using both the remote and the local strategy.
+ ExecException exceptionDuringExecution = null;
+ DynamicExecutionResult dynamicExecutionResult =
+ DynamicExecutionResult.create(
+ StrategyIdentifier.NONE, null, null, /*spawnResults=*/ ImmutableList.of());
+
+ // As an invariant in Bazel, all actions must terminate before the build ends. We use a
+ // synchronizer here, in the main thread, to wait for the termination of both local and remote
+ // spawns. Termination implies successful completion, failure, or, if one spawn wins,
+ // cancellation by the executor.
+ //
+ // In the case where one task completes successfully before the other starts, Bazel must
+ // proceed and return, skipping the other spawn. To achieve this, we use Phaser for its ability
+ // to register a variable number of tasks.
+ //
+ // TODO(b/118451841): Note that this may incur a performance issue where a remote spawn is
+ // faster than a worker spawn, because the worker spawn cannot be cancelled once it starts. This
+ // nullifies the gains from the faster spawn.
+ Phaser bothTasksFinished = new Phaser(/*parties=*/ 1);
+
+ try {
+ final AtomicReference<Class<? extends SpawnActionContext>> outputsHaveBeenWritten =
+ new AtomicReference<>(null);
+ dynamicExecutionResult =
+ executorService.invokeAny(
+ ImmutableList.of(
+ new DynamicExecutionCallable(
+ bothTasksFinished,
+ StrategyIdentifier.LOCAL,
+ actionExecutionContext.getFileOutErr()) {
+ @Override
+ List<SpawnResult> callImpl() throws InterruptedException, ExecException {
+ // This is a rather simple approach to make it possible to score a cache hit
+ // on remote execution before even trying to start the action locally. This
+ // saves resources that would otherwise be wasted by continuously starting and
+ // immediately killing local processes. One possibility for improvement would
+ // be to establish a reporting mechanism from strategies back to here, where
+ // we delay starting locally until the remote strategy tells us that the
+ // action isn't a cache hit.
+ if (delayLocalExecution.get()) {
+ Thread.sleep(options.localExecutionDelay);
+ }
+ return runLocally(
+ spawn,
+ actionExecutionContext.withFileOutErr(fileOutErr),
+ outputsHaveBeenWritten);
+ }
+ },
+ new DynamicExecutionCallable(
+ bothTasksFinished,
+ StrategyIdentifier.REMOTE,
+ actionExecutionContext.getFileOutErr()) {
+ @Override
+ public List<SpawnResult> callImpl() throws InterruptedException, ExecException {
+ List<SpawnResult> spawnResults =
+ runRemotely(
+ spawn,
+ actionExecutionContext.withFileOutErr(fileOutErr),
+ outputsHaveBeenWritten);
+ delayLocalExecution.set(true);
+ return spawnResults;
+ }
+ }));
+ } catch (ExecutionException e) {
+ Throwables.propagateIfPossible(e.getCause(), InterruptedException.class);
+ // DynamicExecutionCallable.callImpl only declares InterruptedException, so this should never
+ // happen.
+ exceptionDuringExecution = new UserExecException(e.getCause());
+ } finally {
+ bothTasksFinished.arriveAndAwaitAdvance();
+ if (dynamicExecutionResult.execException() != null) {
+ exceptionDuringExecution = dynamicExecutionResult.execException();
+ }
+ if (Thread.currentThread().isInterrupted()) {
+ // Warn but don't throw, in case we're crashing.
+ logger.warning("Interrupted waiting for dynamic execution tasks to finish");
+ }
+ }
+ // Check for interruption outside of finally block, so we don't mask any other exceptions.
+ // Clear the interrupt bit if it's set.
+ if (exceptionDuringExecution == null && Thread.interrupted()) {
+ throw new InterruptedException("Interrupted waiting for dynamic execution tasks to finish");
+ }
+ StrategyIdentifier winningStrategy = dynamicExecutionResult.strategyIdentifier();
+ FileOutErr fileOutErr = dynamicExecutionResult.fileOutErr();
+ if (StrategyIdentifier.NONE.equals(winningStrategy) || fileOutErr == null) {
+ throw new IllegalStateException("Neither local or remote execution has started.");
+ }
+
+ try {
+ moveFileOutErr(actionExecutionContext, fileOutErr);
+ } catch (IOException e) {
+ String strategyName = winningStrategy.name().toLowerCase();
+ if (exceptionDuringExecution == null) {
+ throw new UserExecException(
+ String.format("Could not move action logs from %s execution", strategyName), e);
+ } else {
+ actionExecutionContext
+ .getEventHandler()
+ .handle(
+ Event.warn(
+ String.format(
+ "Could not move action logs from %s execution: %s",
+ strategyName, e.toString())));
+ }
+ }
+
+ if (exceptionDuringExecution != null) {
+ throw exceptionDuringExecution;
+ }
+
+ if (options.debugSpawnScheduler) {
+ actionExecutionContext
+ .getEventHandler()
+ .handle(
+ Event.info(
+ String.format(
+ "%s action %s %s",
+ spawn.getMnemonic(),
+ dynamicExecutionResult.execException() == null ? "finished" : "failed",
+ winningStrategy.prettyName())));
+ }
+
+ // TODO(b/62588075) If a second list of spawnResults was generated (before execution was
+ // cancelled), then we might want to save it as well (e.g. for metrics purposes).
+ return dynamicExecutionResult.spawnResults();
+ }
+
+ private void moveFileOutErr(ActionExecutionContext actionExecutionContext, FileOutErr outErr)
+ throws IOException {
+ if (outErr.getOutputPath().exists()) {
+ Files.move(
+ outErr.getOutputPath().getPathFile(),
+ actionExecutionContext.getFileOutErr().getOutputPath().getPathFile());
+ }
+ if (outErr.getErrorPath().exists()) {
+ Files.move(
+ outErr.getErrorPath().getPathFile(),
+ actionExecutionContext.getFileOutErr().getErrorPath().getPathFile());
+ }
+ }
+
+ private static FileOutErr getSuffixedFileOutErr(FileOutErr fileOutErr, String suffix) {
+ Path outDir = Preconditions.checkNotNull(fileOutErr.getOutputPath().getParentDirectory());
+ String outBaseName = fileOutErr.getOutputPath().getBaseName();
+ Path errDir = Preconditions.checkNotNull(fileOutErr.getErrorPath().getParentDirectory());
+ String errBaseName = fileOutErr.getErrorPath().getBaseName();
+ return new FileOutErr(
+ outDir.getChild(outBaseName + suffix), errDir.getChild(errBaseName + suffix));
+ }
+
+ private List<SpawnResult> runLocally(
+ Spawn spawn,
+ ActionExecutionContext actionExecutionContext,
+ AtomicReference<Class<? extends SpawnActionContext>> outputWriteBarrier)
+ throws ExecException, InterruptedException {
+ SandboxedSpawnActionContext strategy;
+ if (!WORKER_BLACKLISTED_MNEMONICS.contains(spawn.getMnemonic())
+ && "1".equals(spawn.getExecutionInfo().get(ExecutionRequirements.SUPPORTS_WORKERS))) {
+ strategy = Preconditions.checkNotNull(workerStrategy, "executorCreated not yet called");
+ } else {
+ strategy = Preconditions.checkNotNull(localStrategy, "executorCreated not yet called");
+ }
+ return strategy.exec(spawn, actionExecutionContext, outputWriteBarrier);
+ }
+
+ private List<SpawnResult> runRemotely(
+ Spawn spawn,
+ ActionExecutionContext actionExecutionContext,
+ AtomicReference<Class<? extends SpawnActionContext>> outputWriteBarrier)
+ throws ExecException, InterruptedException {
+ SandboxedSpawnActionContext strategy =
+ Preconditions.checkNotNull(remoteStrategy, "executorCreated not yet called");
+ return strategy.exec(spawn, actionExecutionContext, outputWriteBarrier);
+ }
+
+ private abstract static class DynamicExecutionCallable
+ implements Callable<DynamicExecutionResult> {
+ private final Phaser taskFinished;
+ private final StrategyIdentifier strategyIdentifier;
+ protected final FileOutErr fileOutErr;
+
+ DynamicExecutionCallable(
+ Phaser taskFinished,
+ StrategyIdentifier strategyIdentifier,
+ FileOutErr fileOutErr) {
+ this.taskFinished = taskFinished;
+ this.strategyIdentifier = strategyIdentifier;
+ this.fileOutErr = getSuffixedFileOutErr(fileOutErr, "." + strategyIdentifier.name());
+ }
+
+ abstract List<SpawnResult> callImpl() throws InterruptedException, ExecException;
+
+ @Override
+ public final DynamicExecutionResult call() throws InterruptedException {
+ taskFinished.register();
+ try {
+ List<SpawnResult> spawnResults = callImpl();
+ return DynamicExecutionResult.create(strategyIdentifier, fileOutErr, null, spawnResults);
+ } catch (Exception e) {
+ Throwables.throwIfInstanceOf(e, InterruptedException.class);
+ return DynamicExecutionResult.create(
+ strategyIdentifier,
+ fileOutErr, e instanceof ExecException ? (ExecException) e : new UserExecException(e),
+ /*spawnResults=*/ ImmutableList.of());
+ } finally {
+ try {
+ fileOutErr.close();
+ } catch (IOException ignored) {
+ // Nothing we can do here.
+ }
+ taskFinished.arriveAndDeregister();
+ }
+ }
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionPolicy.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionPolicy.java
index 40bbf38..b7265f8 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/ExecutionPolicy.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionPolicy.java
@@ -37,10 +37,18 @@
this.locality = locality;
}
+ public boolean canRunRemotelyOnly() {
+ return locality == Locality.REMOTE_ONLY;
+ }
+
public boolean canRunRemotely() {
return locality != Locality.LOCAL_ONLY;
}
+ public boolean canRunLocallyOnly() {
+ return locality == Locality.LOCAL_ONLY;
+ }
+
public boolean canRunLocally() {
return locality != Locality.REMOTE_ONLY;
}
diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index c2532ce..d4e6665 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -242,6 +242,29 @@
)
java_test(
+ name = "dynamic_test",
+ size = "small",
+ srcs = glob(["dynamic/*.java"]),
+ tags = [
+ "no_windows",
+ ],
+ test_class = "com.google.devtools.build.lib.AllTests",
+ deps = [
+ ":actions_testutil",
+ ":foundations_testutil",
+ ":guava_junit_truth",
+ ":testutil",
+ "//src/main/java/com/google/devtools/build/lib:build-base",
+ "//src/main/java/com/google/devtools/build/lib:io",
+ "//src/main/java/com/google/devtools/build/lib/actions",
+ "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
+ "//src/main/java/com/google/devtools/build/lib/dynamic",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
+ "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+ ],
+)
+
+java_test(
name = "events_test",
size = "small",
srcs = glob(["events/*.java"]),
diff --git a/src/test/java/com/google/devtools/build/lib/dynamic/DynamicSpawnStrategyTest.java b/src/test/java/com/google/devtools/build/lib/dynamic/DynamicSpawnStrategyTest.java
new file mode 100644
index 0000000..1f3454b
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/dynamic/DynamicSpawnStrategyTest.java
@@ -0,0 +1,612 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.dynamic;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.fail;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionMetadata;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionKeyContext;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.ArtifactRoot;
+import com.google.devtools.build.lib.actions.BaseSpawn;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.ExecutorInitException;
+import com.google.devtools.build.lib.actions.LocalHostCapacity;
+import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.SandboxedSpawnActionContext;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.actions.SpawnResult;
+import com.google.devtools.build.lib.actions.UserExecException;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
+import com.google.devtools.build.lib.actions.util.ActionsTestUtil.NullAction;
+import com.google.devtools.build.lib.exec.ExecutionPolicy;
+import com.google.devtools.build.lib.testutil.TestThread;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Root;
+import com.google.devtools.build.lib.vfs.util.FileSystems;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link DynamicSpawnStrategy}. */
+@RunWith(JUnit4.class)
+public class DynamicSpawnStrategyTest {
+ protected FileSystem fileSystem;
+ protected Path testRoot;
+ private ExecutorService executorService;
+ private MockLocalSpawnStrategy localStrategy;
+ private MockRemoteSpawnStrategy remoteStrategy;
+ private SpawnActionContext dynamicSpawnStrategy;
+ private Artifact inputArtifact;
+ private Artifact outputArtifact;
+ private FileOutErr outErr;
+ private ActionExecutionContext actionExecutionContext;
+ private DynamicExecutionOptions options;
+ private final ActionKeyContext actionKeyContext = new ActionKeyContext();
+
+ abstract static class MockSpawnStrategy implements SandboxedSpawnActionContext {
+ private final Path testRoot;
+ private final int delayMs;
+ private volatile Spawn executedSpawn;
+ private CountDownLatch succeeded = new CountDownLatch(1);
+ private boolean failsDuringExecution;
+ private CountDownLatch beforeExecutionWaitFor;
+ private Callable<List<SpawnResult>> execute;
+
+ public MockSpawnStrategy(Path testRoot, int delayMs) {
+ this.testRoot = testRoot;
+ this.delayMs = delayMs;
+ }
+
+ @Override
+ public List<SpawnResult> exec(Spawn spawn, ActionExecutionContext actionExecutionContext)
+ throws ExecException, InterruptedException {
+ return exec(spawn, actionExecutionContext, null);
+ }
+
+ @Override
+ public List<SpawnResult> exec(
+ Spawn spawn,
+ ActionExecutionContext actionExecutionContext,
+ AtomicReference<Class<? extends SpawnActionContext>> writeOutputFiles)
+ throws ExecException, InterruptedException {
+ executedSpawn = spawn;
+
+ if (beforeExecutionWaitFor != null) {
+ beforeExecutionWaitFor.countDown();
+ beforeExecutionWaitFor.await();
+ }
+
+ if (delayMs > 0) {
+ Thread.sleep(delayMs);
+ }
+
+ List<SpawnResult> spawnResults = ImmutableList.of();
+ if (execute != null) {
+ try {
+ spawnResults = execute.call();
+ } catch (ExecException | InterruptedException e) {
+ throw e;
+ } catch (Exception e) {
+ Throwables.throwIfUnchecked(e);
+ throw new IllegalStateException(e);
+ }
+ }
+ if (failsDuringExecution) {
+ try {
+ FileSystemUtils.appendIsoLatin1(
+ actionExecutionContext.getFileOutErr().getOutputPath(),
+ "action failed with " + getClass().getSimpleName());
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ throw new UserExecException(getClass().getSimpleName() + " failed to execute the Spawn");
+ }
+
+ if (writeOutputFiles != null && !writeOutputFiles.compareAndSet(null, getClass())) {
+ throw new InterruptedException(getClass() + " could not acquire barrier");
+ } else {
+ for (ActionInput output : spawn.getOutputFiles()) {
+ try {
+ FileSystemUtils.writeIsoLatin1(
+ testRoot.getRelative(output.getExecPath()), getClass().getSimpleName());
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+
+ try {
+ FileSystemUtils.appendIsoLatin1(
+ actionExecutionContext.getFileOutErr().getOutputPath(),
+ "output files written with " + getClass().getSimpleName());
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
+
+ succeeded.countDown();
+
+ return spawnResults;
+ }
+
+ public Spawn getExecutedSpawn() {
+ return executedSpawn;
+ }
+
+ boolean succeeded() {
+ return succeeded.getCount() == 0;
+ }
+
+ CountDownLatch getSucceededLatch() {
+ return succeeded;
+ }
+
+ public void failsDuringExecution() {
+ failsDuringExecution = true;
+ }
+
+ public void beforeExecutionWaitFor(CountDownLatch countDownLatch) {
+ beforeExecutionWaitFor = countDownLatch;
+ }
+
+ void setExecute(Callable<List<SpawnResult>> execute) {
+ this.execute = execute;
+ }
+ }
+
+ @ExecutionStrategy(
+ name = {"mock-remote"},
+ contextType = SpawnActionContext.class
+ )
+ private static class MockRemoteSpawnStrategy extends MockSpawnStrategy {
+ public MockRemoteSpawnStrategy(Path testRoot, int delayMs) {
+ super(testRoot, delayMs);
+ }
+ }
+
+ @ExecutionStrategy(
+ name = {"mock-local"},
+ contextType = SpawnActionContext.class
+ )
+ private static class MockLocalSpawnStrategy extends MockSpawnStrategy {
+ public MockLocalSpawnStrategy(Path testRoot, int delayMs) {
+ super(testRoot, delayMs);
+ }
+ }
+
+ private static class DynamicSpawnStrategyUnderTest extends DynamicSpawnStrategy {
+ public DynamicSpawnStrategyUnderTest(
+ ExecutorService executorService,
+ DynamicExecutionOptions options,
+ Function<Spawn, ExecutionPolicy> executionPolicy) {
+ super(executorService, options, executionPolicy);
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ ResourceManager.instance().setAvailableResources(LocalHostCapacity.getLocalHostCapacity());
+ ResourceManager.instance()
+ .setRamUtilizationPercentage(ResourceManager.DEFAULT_RAM_UTILIZATION_PERCENTAGE);
+ ResourceManager.instance().resetResourceUsage();
+
+ fileSystem = FileSystems.getNativeFileSystem();
+ testRoot = fileSystem.getPath(TestUtils.tmpDir());
+ FileSystemUtils.deleteTreesBelow(testRoot);
+ executorService = Executors.newCachedThreadPool();
+ inputArtifact =
+ new Artifact(
+ PathFragment.create("input.txt"), ArtifactRoot.asSourceRoot(Root.fromPath(testRoot)));
+ outputArtifact =
+ new Artifact(
+ PathFragment.create("output.txt"), ArtifactRoot.asSourceRoot(Root.fromPath(testRoot)));
+ outErr = new FileOutErr(testRoot.getRelative("stdout"), testRoot.getRelative("stderr"));
+ actionExecutionContext =
+ ActionsTestUtil.createContext(null, actionKeyContext, outErr, testRoot, null, null);
+ }
+
+ void createSpawnStrategy(int localDelay, int remoteDelay) throws ExecutorInitException {
+ localStrategy = new MockLocalSpawnStrategy(testRoot, localDelay);
+ remoteStrategy = new MockRemoteSpawnStrategy(testRoot, remoteDelay);
+ options = new DynamicExecutionOptions();
+ options.dynamicLocalStrategy = "mock-local";
+ options.dynamicRemoteStrategy = "mock-remote";
+ options.dynamicWorkerStrategy = "mock-local";
+ options.internalSpawnScheduler = true;
+ options.localExecutionDelay = 0;
+ dynamicSpawnStrategy =
+ new DynamicSpawnStrategyUnderTest(executorService, options, this::getExecutionPolicy);
+ dynamicSpawnStrategy.executorCreated(ImmutableList.of(localStrategy, remoteStrategy));
+ }
+
+ ExecutionPolicy getExecutionPolicy(Spawn spawn) {
+ if (spawn.getExecutionInfo().containsKey("local")) {
+ return ExecutionPolicy.LOCAL_EXECUTION_ONLY;
+ } else if (spawn.getExecutionInfo().containsKey("remote")) {
+ return ExecutionPolicy.REMOTE_EXECUTION_ONLY;
+ } else {
+ return ExecutionPolicy.ANYWHERE;
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ executorService.shutdownNow();
+ }
+
+ Spawn getSpawnForTest(boolean forceLocal, boolean forceRemote) {
+ Preconditions.checkArgument(
+ !(forceLocal && forceRemote), "Cannot force local and remote at the same time");
+ ActionExecutionMetadata action =
+ new NullAction(ImmutableList.of(inputArtifact), outputArtifact);
+ return new BaseSpawn(
+ ImmutableList.<String>of(),
+ ImmutableMap.<String, String>of(),
+ forceLocal
+ ? ImmutableMap.of("local", "1")
+ : forceRemote ? ImmutableMap.of("remote", "1") : ImmutableMap.<String, String>of(),
+ action,
+ ResourceSet.create(1, 0, 0));
+ }
+
+ @Test
+ public void nonRemotableSpawnRunsLocally() throws Exception {
+ Spawn spawn = getSpawnForTest(true, false);
+ createSpawnStrategy(0, 0);
+
+ dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
+
+ assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(localStrategy.succeeded()).isTrue();
+ assertThat(remoteStrategy.getExecutedSpawn()).isNull();
+ assertThat(remoteStrategy.succeeded()).isFalse();
+
+ assertThat(outErr.outAsLatin1()).contains("output files written with MockLocalSpawnStrategy");
+ assertThat(outErr.outAsLatin1()).doesNotContain("MockRemoteSpawnStrategy");
+ }
+
+ @Test
+ public void nonLocallyExecutableSpawnRunsRemotely() throws Exception {
+ Spawn spawn = getSpawnForTest(false, true);
+ createSpawnStrategy(0, 0);
+
+ dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
+
+ assertThat(localStrategy.getExecutedSpawn()).isNull();
+ assertThat(localStrategy.succeeded()).isFalse();
+ assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(remoteStrategy.succeeded()).isTrue();
+
+ assertThat(outErr.outAsLatin1()).contains("output files written with MockRemoteSpawnStrategy");
+ assertThat(outErr.outAsLatin1()).doesNotContain("MockLocalSpawnStrategy");
+ }
+
+ @Test
+ public void actionSucceedsIfLocalExecutionSucceedsEvenIfRemoteFailsLater() throws Exception {
+ Spawn spawn = getSpawnForTest(false, false);
+ createSpawnStrategy(0, 2000);
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+ localStrategy.beforeExecutionWaitFor(countDownLatch);
+ remoteStrategy.beforeExecutionWaitFor(countDownLatch);
+ remoteStrategy.failsDuringExecution();
+
+ dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
+
+ assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(localStrategy.succeeded()).isTrue();
+ assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(remoteStrategy.succeeded()).isFalse();
+
+ assertThat(outErr.outAsLatin1()).contains("output files written with MockLocalSpawnStrategy");
+ assertThat(outErr.outAsLatin1()).doesNotContain("MockRemoteSpawnStrategy");
+ }
+
+ @Test
+ public void actionSucceedsIfRemoteExecutionSucceedsEvenIfLocalFailsLater() throws Exception {
+ Spawn spawn = getSpawnForTest(false, false);
+ createSpawnStrategy(2000, 0);
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+ localStrategy.beforeExecutionWaitFor(countDownLatch);
+ localStrategy.failsDuringExecution();
+ remoteStrategy.beforeExecutionWaitFor(countDownLatch);
+
+ dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
+
+ assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(localStrategy.succeeded()).isFalse();
+ assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(remoteStrategy.succeeded()).isTrue();
+
+ assertThat(outErr.outAsLatin1()).contains("output files written with MockRemoteSpawnStrategy");
+ assertThat(outErr.outAsLatin1()).doesNotContain("MockLocalSpawnStrategy");
+ }
+
+ @Test
+ public void actionFailsIfLocalFailsImmediatelyEvenIfRemoteSucceedsLater() throws Exception {
+ Spawn spawn = getSpawnForTest(false, false);
+ createSpawnStrategy(0, 2000);
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+ localStrategy.beforeExecutionWaitFor(countDownLatch);
+ localStrategy.failsDuringExecution();
+ remoteStrategy.beforeExecutionWaitFor(countDownLatch);
+
+ try {
+ dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
+ fail("Expected dynamicSpawnStrategy to throw an ExecException");
+ } catch (ExecException e) {
+ assertThat(e).hasMessageThat().matches("MockLocalSpawnStrategy failed to execute the Spawn");
+ }
+
+ assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(localStrategy.succeeded()).isFalse();
+ assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(remoteStrategy.succeeded()).isFalse();
+
+ assertThat(outErr.outAsLatin1()).contains("action failed with MockLocalSpawnStrategy");
+ assertThat(outErr.outAsLatin1()).doesNotContain("MockRemoteSpawnStrategy");
+ }
+
+ @Test
+ public void actionFailsIfRemoteFailsImmediatelyEvenIfLocalSucceedsLater() throws Exception {
+ Spawn spawn = getSpawnForTest(false, false);
+ createSpawnStrategy(2000, 0);
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+ localStrategy.beforeExecutionWaitFor(countDownLatch);
+ remoteStrategy.beforeExecutionWaitFor(countDownLatch);
+ remoteStrategy.failsDuringExecution();
+
+ try {
+ dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
+ fail("Expected dynamicSpawnStrategy to throw an ExecException");
+ } catch (ExecException e) {
+ assertThat(e).hasMessageThat().matches("MockRemoteSpawnStrategy failed to execute the Spawn");
+ }
+
+ assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(localStrategy.succeeded()).isFalse();
+ assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(remoteStrategy.succeeded()).isFalse();
+
+ assertThat(outErr.outAsLatin1()).contains("action failed with MockRemoteSpawnStrategy");
+ assertThat(outErr.outAsLatin1()).doesNotContain("MockLocalSpawnStrategy");
+ }
+
+ @Test
+ public void actionFailsIfLocalAndRemoteFail() throws Exception {
+ Spawn spawn = getSpawnForTest(false, false);
+ createSpawnStrategy(0, 0);
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+ localStrategy.beforeExecutionWaitFor(countDownLatch);
+ remoteStrategy.beforeExecutionWaitFor(countDownLatch);
+ localStrategy.failsDuringExecution();
+ remoteStrategy.failsDuringExecution();
+
+ try {
+ dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
+ fail("Expected dynamicSpawnStrategy to throw an ExecException");
+ } catch (ExecException e) {
+ assertThat(e)
+ .hasMessageThat()
+ .matches("Mock(Local|Remote)SpawnStrategy failed to execute the Spawn");
+ }
+
+ assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(localStrategy.succeeded()).isFalse();
+ assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(remoteStrategy.succeeded()).isFalse();
+ }
+
+ @Test
+ public void noDeadlockWithSingleThreadedExecutor() throws Exception {
+ final Spawn spawn = getSpawnForTest(/*forceLocal=*/ false, /*forceRemote=*/ false);
+
+ // Replace the executorService with a single threaded one.
+ executorService = Executors.newSingleThreadExecutor();
+ createSpawnStrategy(/*localDelay=*/ 0, /*remoteDelay=*/ 0);
+
+ dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
+
+ assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
+ assertThat(localStrategy.succeeded()).isTrue();
+
+ /**
+ * The single-threaded executorService#invokeAny does not comply to the contract where
+ * the callables are *always* called sequentially. In this case, both spawns will start
+ * executing, but the local one will always succeed as it's the first to be called. The remote
+ * one will then be cancelled, or is null if the local one completes before the remote one
+ * starts.
+ *
+ * See the documentation of {@link BoundedExectorService#invokeAny(Collection)}, specifically:
+ * "The following is less efficient (it goes on submitting tasks even if there is some task
+ * already finished), but quite straight-forward.".
+ */
+ assertThat(remoteStrategy.getExecutedSpawn()).isAnyOf(spawn, null);
+ assertThat(remoteStrategy.succeeded()).isFalse();
+ }
+
+ @Test
+ public void interruptDuringExecutionDoesActuallyInterruptTheExecution() throws Exception {
+ final Spawn spawn = getSpawnForTest(false, false);
+ createSpawnStrategy(60000, 60000);
+ CountDownLatch countDownLatch = new CountDownLatch(2);
+ localStrategy.beforeExecutionWaitFor(countDownLatch);
+ remoteStrategy.beforeExecutionWaitFor(countDownLatch);
+
+ TestThread testThread =
+ new TestThread() {
+ @Override
+ public void runTest() throws Exception {
+ try {
+ dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
+ } catch (InterruptedException e) {
+ // This is expected.
+ }
+ }
+ };
+ testThread.start();
+ countDownLatch.await(5, TimeUnit.SECONDS);
+ testThread.interrupt();
+ testThread.joinAndAssertState(5000);
+
+ assertThat(outErr.getOutputPath().exists()).isFalse();
+ assertThat(outErr.getErrorPath().exists()).isFalse();
+ }
+
+ private void strategyWaitsForBothSpawnsToFinish(boolean interruptThread, boolean executionFails)
+ throws Exception {
+ final Spawn spawn = getSpawnForTest(false, false);
+ createSpawnStrategy(0, 0);
+ CountDownLatch waitToFinish = new CountDownLatch(1);
+ CountDownLatch wasInterrupted = new CountDownLatch(1);
+ CountDownLatch executionCanProceed = new CountDownLatch(2);
+ localStrategy.setExecute(
+ () -> {
+ executionCanProceed.countDown();
+ try {
+ Thread.sleep(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
+ throw new IllegalStateException("Should have been interrupted");
+ } catch (InterruptedException e) {
+ // Expected.
+ }
+ wasInterrupted.countDown();
+ try {
+ Preconditions.checkState(
+ waitToFinish.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
+ } catch (InterruptedException e) {
+ throw new IllegalStateException(e);
+ }
+ return ImmutableList.of();
+ });
+ if (executionFails) {
+ remoteStrategy.failsDuringExecution();
+ }
+ remoteStrategy.beforeExecutionWaitFor(executionCanProceed);
+
+ TestThread testThread =
+ new TestThread() {
+ @Override
+ public void runTest() {
+ try {
+ dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
+ Preconditions.checkState(!interruptThread && !executionFails);
+ } catch (InterruptedException e) {
+ Preconditions.checkState(interruptThread && !executionFails);
+ Preconditions.checkState(!Thread.currentThread().isInterrupted());
+ } catch (ExecException e) {
+ Preconditions.checkState(executionFails);
+ Preconditions.checkState(Thread.currentThread().isInterrupted() == interruptThread);
+ }
+ }
+ };
+ testThread.start();
+ if (!executionFails) {
+ assertThat(
+ remoteStrategy
+ .getSucceededLatch()
+ .await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+ .isTrue();
+ }
+ assertThat(wasInterrupted.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)).isTrue();
+ assertThat(testThread.isAlive()).isTrue();
+ if (interruptThread) {
+ testThread.interrupt();
+ }
+ // Wait up to 5 seconds for this thread to finish. It should not have finished.
+ testThread.join(5000);
+ assertThat(testThread.isAlive()).isTrue();
+ waitToFinish.countDown();
+ testThread.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
+ }
+
+ @Test
+ public void strategyWaitsForBothSpawnsToFinish() throws Exception {
+ strategyWaitsForBothSpawnsToFinish(false, false);
+ }
+
+ @Test
+ public void strategyWaitsForBothSpawnsToFinishEvenIfInterrupted() throws Exception {
+ strategyWaitsForBothSpawnsToFinish(true, false);
+ }
+
+ @Test
+ public void strategyWaitsForBothSpawnsToFinishOnFailure() throws Exception {
+ strategyWaitsForBothSpawnsToFinish(false, true);
+ }
+
+ @Test
+ public void strategyWaitsForBothSpawnsToFinishOnFailureEvenIfInterrupted() throws Exception {
+ strategyWaitsForBothSpawnsToFinish(true, true);
+ }
+
+ @Test
+ public void strategyPropagatesFasterLocalException() throws Exception {
+ strategyPropagatesException(true);
+ }
+
+ @Test
+ public void strategyPropagatesFasterRemoteException() throws Exception {
+ strategyPropagatesException(false);
+ }
+
+ private void strategyPropagatesException(boolean preferLocal) throws Exception {
+ final Spawn spawn = getSpawnForTest(false, false);
+ createSpawnStrategy(!preferLocal ? 60000 : 0, preferLocal ? 60000 : 0);
+
+ String message = "Mock spawn execution exception";
+ Callable<List<SpawnResult>> execute = () -> {
+ throw new IllegalStateException(message);
+ };
+ localStrategy.setExecute(execute);
+ remoteStrategy.setExecute(execute);
+
+ try {
+ dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
+ fail("Expected dynamicSpawnStrategy to throw an ExecException");
+ } catch (ExecException e) {
+ assertThat(e).hasMessageThat().matches("java.lang.IllegalStateException: " + message);
+ }
+
+ Spawn executedSpawn = localStrategy.getExecutedSpawn();
+ executedSpawn = executedSpawn == null ? remoteStrategy.getExecutedSpawn() : executedSpawn;
+ assertThat(executedSpawn).isEqualTo(spawn);
+ assertThat(localStrategy.succeeded()).isFalse();
+ assertThat(remoteStrategy.succeeded()).isFalse();
+ }
+}
