// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.dynamic;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.truth.Truth.assertThat;
import static com.google.devtools.build.lib.testutil.MoreAsserts.assertThrows;
import static junit.framework.TestCase.fail;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.Uninterruptibles;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionExecutionMetadata;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionKeyContext;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactRoot;
import com.google.devtools.build.lib.actions.BaseSpawn;
import com.google.devtools.build.lib.actions.EmptyRunfilesSupplier;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.ExecutionStrategy;
import com.google.devtools.build.lib.actions.ExecutorInitException;
import com.google.devtools.build.lib.actions.ResourceSet;
import com.google.devtools.build.lib.actions.SandboxedSpawnActionContext;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnActionContext;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
import com.google.devtools.build.lib.actions.util.ActionsTestUtil.NullAction;
import com.google.devtools.build.lib.exec.ExecutionPolicy;
import com.google.devtools.build.lib.testutil.TestThread;
import com.google.devtools.build.lib.testutil.TestUtils;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.util.FileSystems;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Logger;
import javax.annotation.Nullable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;

/** Tests for {@link DynamicSpawnStrategy}. */
@RunWith(Parameterized.class)
public class DynamicSpawnStrategyTest {
  private Path testRoot;
  private ExecutorService executorServiceForCleanup;
  private FileOutErr outErr;
  private ActionExecutionContext actionExecutionContext;
  private final ActionKeyContext actionKeyContext = new ActionKeyContext();

  @Parameters(name = "{index}: legacy={0}")
  public static Collection<Object[]> data() {
    return Arrays.asList(
        new Object[][] {
          {true}, {false},
        });
  }

  @Parameter public boolean legacyBehavior;

  private SpawnActionContext newDynamicSpawnStrategy(
      ExecutorService executorService, DynamicExecutionOptions options) {
    if (legacyBehavior) {
      return new LegacyDynamicSpawnStrategy(
          executorService, options, DynamicSpawnStrategyTest::getExecutionPolicy);
    } else {
      return new DynamicSpawnStrategy(
          executorService, options, DynamicSpawnStrategyTest::getExecutionPolicy);
    }
  }

  /** Syntactic sugar to decrease and await for a latch in a single line. */
  private static void countDownAndWait(CountDownLatch countDownLatch) throws InterruptedException {
    countDownLatch.countDown();
    countDownLatch.await();
  }

  /** Hook to implement per-test custom logic in the {@link MockSpawnStrategy}. */
  @FunctionalInterface
  interface DoExec {
    DoExec NOTHING = (self, spawn, actionExecutionContext) -> {};

    void run(MockSpawnStrategy self, Spawn spawn, ActionExecutionContext actionExecutionContext)
        throws ExecException, InterruptedException;
  }

  /**
   * Minimal implementation of a strategy for testing purposes.
   *
   * <p>All the logic in here must be applicable to all tests. If any test needs to special-case
   * some aspect of this logic, then it must extend this subclass as necessary.
   *
   * <p>Note, however, that this class lacks {@link ExecutionStrategy} annotations and is the main
   * reason why this is marked as abstract. You must use any of the direct subclasses of this
   * strategy instead (optionally extending them).
   */
  private abstract static class MockSpawnStrategy implements SandboxedSpawnActionContext {
    /** Identifier of this class for error reporting purposes. */
    private final String name;

    /** Base location of the file hierarchy where the stdout/stderr of the spawn is written to. */
    private final Path testRoot;

    /** Lazily set to the spawn passed to {@link #exec} as soon as that hook is invoked. */
    @Nullable private volatile Spawn executedSpawn;

    /** Tracks whether {@link #exec} completed successfully or not. */
    private CountDownLatch succeeded = new CountDownLatch(1);

    /** Hook to implement per-test custom logic. */
    private final DoExec doExecBeforeStop;

    private final DoExec doExecAfterStop;

    MockSpawnStrategy(String name, Path testRoot, DoExec doExecBeforeStop, DoExec doExecAfterStop) {
      this.name = name;
      this.testRoot = testRoot;
      this.doExecBeforeStop = doExecBeforeStop;
      this.doExecAfterStop = doExecAfterStop;
    }

    /** Helper to record an execution failure from within {@link #doExecBeforeStop}. */
    void failExecution(ActionExecutionContext actionExecutionContext) throws ExecException {
      try {
        FileSystemUtils.appendIsoLatin1(
            actionExecutionContext.getFileOutErr().getOutputPath(), "action failed with " + name);
      } catch (IOException e) {
        throw new IllegalStateException(e);
      }
      throw new UserExecException(name + " failed to execute the Spawn");
    }

    @Override
    public List<SpawnResult> exec(
        Spawn spawn,
        ActionExecutionContext actionExecutionContext,
        @Nullable StopConcurrentSpawns stopConcurrentSpawns)
        throws ExecException, InterruptedException {
      executedSpawn = spawn;

      doExecBeforeStop.run(this, spawn, actionExecutionContext);
      if (stopConcurrentSpawns != null) {
        stopConcurrentSpawns.stop();
        doExecAfterStop.run(this, spawn, actionExecutionContext);
      }

      for (ActionInput output : spawn.getOutputFiles()) {
        try {
          FileSystemUtils.writeIsoLatin1(testRoot.getRelative(output.getExecPath()), name);
        } catch (IOException e) {
          throw new IllegalStateException(e);
        }
      }

      try {
        FileSystemUtils.appendIsoLatin1(
            actionExecutionContext.getFileOutErr().getOutputPath(),
            "output files written with " + name);
      } catch (IOException e) {
        throw new IllegalStateException(e);
      }

      succeeded.countDown();

      return ImmutableList.of();
    }

    @Override
    public List<SpawnResult> exec(Spawn spawn, ActionExecutionContext actionExecutionContext) {
      throw new IllegalStateException("Not expected to be called");
    }

    @Override
    public boolean canExec(Spawn spawn) {
      return true;
    }

    @Nullable
    public Spawn getExecutedSpawn() {
      return executedSpawn;
    }

    /** Returns true if {@link #exec} was called and completed successfully; does not block. */
    boolean succeeded() {
      return succeeded.getCount() == 0;
    }
  }

  /** Extends the mock strategy with an execution strategy annotation for remote execution. */
  @ExecutionStrategy(
      name = {"mock-remote"},
      contextType = SpawnActionContext.class)
  private static class MockRemoteSpawnStrategy extends MockSpawnStrategy {
    MockRemoteSpawnStrategy(Path testRoot) {
      this(testRoot, DoExec.NOTHING);
    }

    MockRemoteSpawnStrategy(Path testRoot, DoExec doExecBeforeStop) {
      super("MockRemoteSpawnStrategy", testRoot, doExecBeforeStop, DoExec.NOTHING);
    }

    MockRemoteSpawnStrategy(Path testRoot, DoExec doExecBeforeStop, DoExec doExecAfterStop) {
      super("MockRemoteSpawnStrategy", testRoot, doExecBeforeStop, doExecAfterStop);
    }
  }

  /** Extends the mock strategy with an execution strategy annotation for local execution. */
  @ExecutionStrategy(
      name = {"mock-local"},
      contextType = SpawnActionContext.class)
  private static class MockLocalSpawnStrategy extends MockSpawnStrategy {
    MockLocalSpawnStrategy(Path testRoot) {
      this(testRoot, DoExec.NOTHING);
    }

    MockLocalSpawnStrategy(Path testRoot, DoExec doExecBeforeStop) {
      super("MockLocalSpawnStrategy", testRoot, doExecBeforeStop, DoExec.NOTHING);
    }

    MockLocalSpawnStrategy(Path testRoot, DoExec doExecBeforeStop, DoExec doExecAfterStop) {
      super("MockLocalSpawnStrategy", testRoot, doExecBeforeStop, doExecAfterStop);
    }
  }

  /** Extends the mock strategy with an execution strategy annotation for sandboxed execution. */
  @ExecutionStrategy(
      name = {"mock-sandboxed"},
      contextType = SpawnActionContext.class)
  private static class MockSandboxedSpawnStrategy extends MockSpawnStrategy {
    MockSandboxedSpawnStrategy(Path testRoot) {
      this(testRoot, DoExec.NOTHING);
    }

    MockSandboxedSpawnStrategy(Path testRoot, DoExec doExecBeforeStop) {
      super("MockSandboxedSpawnStrategy", testRoot, doExecBeforeStop, DoExec.NOTHING);
    }

    MockSandboxedSpawnStrategy(Path testRoot, DoExec doExecBeforeStop, DoExec doExecAfterStop) {
      super("MockSandboxedSpawnStrategy", testRoot, doExecBeforeStop, doExecAfterStop);
    }
  }

  @Before
  public void setUp() throws Exception {
    testRoot = FileSystems.getNativeFileSystem().getPath(TestUtils.tmpDir());
    testRoot.deleteTreesBelow();
    outErr = new FileOutErr(testRoot.getRelative("stdout"), testRoot.getRelative("stderr"));
    actionExecutionContext =
        ActionsTestUtil.createContext(
            /*executor=*/ null,
            /*eventHandler=*/ null,
            actionKeyContext,
            outErr,
            testRoot,
            /*metadataHandler=*/ null,
            /*actionGraph=*/ null);
  }

  /**
   * Creates a new dynamic spawn strategy with different strategies for local and remote execution.
   *
   * @param localStrategy the strategy for local execution
   * @param remoteStrategy the strategy for remote execution
   * @param executorService the executor to pass to the dynamic strategy
   * @return the constructed dynamic strategy
   * @throws ExecutorInitException if creating the strategy with the given parameters fails
   */
  private SpawnActionContext createSpawnStrategyWithExecutor(
      MockLocalSpawnStrategy localStrategy,
      MockRemoteSpawnStrategy remoteStrategy,
      ExecutorService executorService)
      throws ExecutorInitException {
    DynamicExecutionOptions options = new DynamicExecutionOptions();
    options.dynamicLocalStrategy =
        Lists.newArrayList(Maps.immutableEntry("", ImmutableList.of("mock-local")));
    options.dynamicRemoteStrategy =
        Lists.newArrayList(Maps.immutableEntry("", ImmutableList.of("mock-remote")));
    options.dynamicWorkerStrategy = "mock-local";
    options.localExecutionDelay = 0;

    checkState(executorServiceForCleanup == null);
    executorServiceForCleanup = executorService;

    DynamicExecutionModule.setDefaultStrategiesByMnemonic(options);
    SpawnActionContext dynamicSpawnStrategy = newDynamicSpawnStrategy(executorService, options);
    dynamicSpawnStrategy.executorCreated(ImmutableList.of(localStrategy, remoteStrategy));
    return dynamicSpawnStrategy;
  }

  /**
   * Creates a new dynamic spawn strategy with different strategies for local and remote execution
   * and a default multi-threaded executor service.
   *
   * @param localStrategy the strategy for local execution
   * @param remoteStrategy the strategy for remote execution
   * @return the constructed dynamic strategy
   * @throws ExecutorInitException if creating the strategy with the given parameters fails
   */
  private SpawnActionContext createSpawnStrategy(
      MockLocalSpawnStrategy localStrategy, MockRemoteSpawnStrategy remoteStrategy)
      throws ExecutorInitException {
    return createSpawnStrategyWithExecutor(
        localStrategy, remoteStrategy, Executors.newCachedThreadPool());
  }

  /**
   * Creates a new dynamic spawn strategy with different strategies for local, remote, and sandboxed
   * execution.
   *
   * <p>TODO(jmmv): This overload should not be necessary now that we do not special-case the
   * handling of sandboxed strategies any longer. Remove once the sandbox-specific flags are gone.
   *
   * @param localStrategy the default strategy for local execution
   * @param remoteStrategy the default strategy for remote execution
   * @param sandboxedStrategy the strategy to use when the mnemonic matches {@code testMnemonic}.
   * @return the constructed dynamic strategy
   * @throws ExecutorInitException if creating the strategy with the given parameters fails
   */
  private SpawnActionContext createSpawnStrategy(
      MockLocalSpawnStrategy localStrategy,
      MockRemoteSpawnStrategy remoteStrategy,
      MockSandboxedSpawnStrategy sandboxedStrategy)
      throws ExecutorInitException {
    DynamicExecutionOptions options = new DynamicExecutionOptions();
    options.dynamicLocalStrategy =
        Lists.newArrayList(
            Maps.immutableEntry("", ImmutableList.of("mock-local")),
            Maps.immutableEntry("testMnemonic", ImmutableList.of("mock-sandboxed")));
    options.dynamicRemoteStrategy =
        Lists.newArrayList(
            Maps.immutableEntry("", ImmutableList.of("mock-remote")),
            Maps.immutableEntry("testMnemonic", ImmutableList.of("mock-sandboxed")));
    options.dynamicWorkerStrategy = "mock-local";
    options.internalSpawnScheduler = true;
    options.localExecutionDelay = 0;

    checkState(executorServiceForCleanup == null);
    executorServiceForCleanup = Executors.newCachedThreadPool();

    DynamicExecutionModule.setDefaultStrategiesByMnemonic(options);
    SpawnActionContext dynamicSpawnStrategy =
        newDynamicSpawnStrategy(executorServiceForCleanup, options);
    dynamicSpawnStrategy.executorCreated(
        ImmutableList.of(localStrategy, remoteStrategy, sandboxedStrategy));
    return dynamicSpawnStrategy;
  }

  private static ExecutionPolicy getExecutionPolicy(Spawn spawn) {
    if (spawn.getExecutionInfo().containsKey("local")) {
      return ExecutionPolicy.LOCAL_EXECUTION_ONLY;
    } else if (spawn.getExecutionInfo().containsKey("remote")) {
      return ExecutionPolicy.REMOTE_EXECUTION_ONLY;
    } else {
      return ExecutionPolicy.ANYWHERE;
    }
  }

  private static class NullActionWithMnemonic extends NullAction {
    private final String mnemonic;

    private NullActionWithMnemonic(String mnemonic, List<Artifact> inputs, Artifact... outputs) {
      super(inputs, outputs);
      this.mnemonic = mnemonic;
    }

    @Override
    public String getMnemonic() {
      return mnemonic;
    }
  }

  @After
  public void tearDown() throws Exception {
    if (executorServiceForCleanup != null) {
      executorServiceForCleanup.shutdownNow();
    }
  }

  /** Constructs a new spawn with a custom mnemonic and execution info. */
  private Spawn newCustomSpawn(String mnemonic, ImmutableMap<String, String> executionInfo) {
    Artifact inputArtifact =
        ActionsTestUtil.createArtifact(
            ArtifactRoot.asSourceRoot(Root.fromPath(testRoot)), "input.txt");
    Artifact outputArtifact =
        ActionsTestUtil.createArtifact(
            ArtifactRoot.asSourceRoot(Root.fromPath(testRoot)), "output.txt");

    ActionExecutionMetadata action =
        new NullActionWithMnemonic(mnemonic, ImmutableList.of(inputArtifact), outputArtifact);
    return new BaseSpawn(
        ImmutableList.of(),
        ImmutableMap.of(),
        executionInfo,
        EmptyRunfilesSupplier.INSTANCE,
        action,
        ResourceSet.create(1, 0, 0));
  }

  /** Constructs a new spawn that can be run locally and remotely with arbitrary settings. */
  private Spawn newDynamicSpawn() {
    return newCustomSpawn("Null", ImmutableMap.of());
  }

  @Test
  public void nonRemotableSpawnRunsLocally() throws Exception {
    MockLocalSpawnStrategy localStrategy = new MockLocalSpawnStrategy(testRoot);
    MockRemoteSpawnStrategy remoteStrategy = new MockRemoteSpawnStrategy(testRoot);
    SpawnActionContext dynamicSpawnStrategy = createSpawnStrategy(localStrategy, remoteStrategy);

    Spawn spawn = newCustomSpawn("Null", ImmutableMap.of("local", "1"));
    dynamicSpawnStrategy.exec(spawn, actionExecutionContext);

    assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(localStrategy.succeeded()).isTrue();
    assertThat(remoteStrategy.getExecutedSpawn()).isNull();
    assertThat(remoteStrategy.succeeded()).isFalse();

    assertThat(outErr.outAsLatin1()).contains("output files written with MockLocalSpawnStrategy");
    assertThat(outErr.outAsLatin1()).doesNotContain("MockRemoteSpawnStrategy");
  }

  @Test
  public void localSpawnUsesStrategyByMnemonicWithWorkerFlagDisabled() throws Exception {
    MockLocalSpawnStrategy localStrategy = new MockLocalSpawnStrategy(testRoot);
    MockRemoteSpawnStrategy remoteStrategy = new MockRemoteSpawnStrategy(testRoot);
    MockSandboxedSpawnStrategy sandboxedStrategy = new MockSandboxedSpawnStrategy(testRoot);
    SpawnActionContext dynamicSpawnStrategy =
        createSpawnStrategy(localStrategy, remoteStrategy, sandboxedStrategy);

    Spawn spawn = newCustomSpawn("testMnemonic", ImmutableMap.of("local", "1"));
    dynamicSpawnStrategy.exec(spawn, actionExecutionContext);

    assertThat(localStrategy.getExecutedSpawn()).isNull();
    assertThat(localStrategy.succeeded()).isFalse();
    assertThat(remoteStrategy.getExecutedSpawn()).isNull();
    assertThat(remoteStrategy.succeeded()).isFalse();
    assertThat(sandboxedStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(sandboxedStrategy.succeeded()).isTrue();

    assertThat(outErr.outAsLatin1())
        .contains("output files written with MockSandboxedSpawnStrategy");
    assertThat(outErr.outAsLatin1()).doesNotContain("MockLocalSpawnStrategy");
    assertThat(outErr.outAsLatin1()).doesNotContain("MockRemoteSpawnStrategy");
  }

  @Test
  public void nonLocallyExecutableSpawnRunsRemotely() throws Exception {
    MockLocalSpawnStrategy localStrategy = new MockLocalSpawnStrategy(testRoot);
    MockRemoteSpawnStrategy remoteStrategy = new MockRemoteSpawnStrategy(testRoot);
    SpawnActionContext dynamicSpawnStrategy = createSpawnStrategy(localStrategy, remoteStrategy);

    Spawn spawn = newCustomSpawn("Null", ImmutableMap.of("remote", "1"));
    dynamicSpawnStrategy.exec(spawn, actionExecutionContext);

    assertThat(localStrategy.getExecutedSpawn()).isNull();
    assertThat(localStrategy.succeeded()).isFalse();
    assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(remoteStrategy.succeeded()).isTrue();

    assertThat(outErr.outAsLatin1()).contains("output files written with MockRemoteSpawnStrategy");
    assertThat(outErr.outAsLatin1()).doesNotContain("MockLocalSpawnStrategy");
  }

  @Test
  public void remoteSpawnUsesStrategyByMnemonic() throws Exception {
    MockLocalSpawnStrategy localStrategy = new MockLocalSpawnStrategy(testRoot);
    MockRemoteSpawnStrategy remoteStrategy = new MockRemoteSpawnStrategy(testRoot);
    MockSandboxedSpawnStrategy sandboxedStrategy = new MockSandboxedSpawnStrategy(testRoot);
    SpawnActionContext dynamicSpawnStrategy =
        createSpawnStrategy(localStrategy, remoteStrategy, sandboxedStrategy);

    Spawn spawn = newCustomSpawn("testMnemonic", ImmutableMap.of("remote", "1"));
    dynamicSpawnStrategy.exec(spawn, actionExecutionContext);

    assertThat(localStrategy.getExecutedSpawn()).isNull();
    assertThat(localStrategy.succeeded()).isFalse();
    assertThat(remoteStrategy.getExecutedSpawn()).isNull();
    assertThat(remoteStrategy.succeeded()).isFalse();
    assertThat(sandboxedStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(sandboxedStrategy.succeeded()).isTrue();

    assertThat(outErr.outAsLatin1())
        .contains("output files written with MockSandboxedSpawnStrategy");
    assertThat(outErr.outAsLatin1()).doesNotContain("MockLocalSpawnStrategy");
    assertThat(outErr.outAsLatin1()).doesNotContain("MockRemoteSpawnStrategy");
  }

  @Test
  public void actionSucceedsIfLocalExecutionSucceedsEvenIfRemoteFailsLater() throws Exception {
    CountDownLatch countDownLatch = new CountDownLatch(2);

    MockLocalSpawnStrategy localStrategy =
        new MockLocalSpawnStrategy(
            testRoot, (self, spawn, actionExecutionContext) -> countDownAndWait(countDownLatch));

    MockRemoteSpawnStrategy remoteStrategy =
        new MockRemoteSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              countDownAndWait(countDownLatch);
              Thread.sleep(2000);
              self.failExecution(actionExecutionContext);
            });

    SpawnActionContext dynamicSpawnStrategy = createSpawnStrategy(localStrategy, remoteStrategy);

    Spawn spawn = newDynamicSpawn();
    dynamicSpawnStrategy.exec(spawn, actionExecutionContext);

    assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(localStrategy.succeeded()).isTrue();
    assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(remoteStrategy.succeeded()).isFalse();

    assertThat(outErr.outAsLatin1()).contains("output files written with MockLocalSpawnStrategy");
    assertThat(outErr.outAsLatin1()).doesNotContain("MockRemoteSpawnStrategy");
  }

  @Test
  public void actionSucceedsIfRemoteExecutionSucceedsEvenIfLocalFailsLater() throws Exception {
    CountDownLatch countDownLatch = new CountDownLatch(2);

    MockLocalSpawnStrategy localStrategy =
        new MockLocalSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              countDownAndWait(countDownLatch);
              Thread.sleep(2000);
              self.failExecution(actionExecutionContext);
            });

    MockRemoteSpawnStrategy remoteStrategy =
        new MockRemoteSpawnStrategy(
            testRoot, (self, spawn, actionExecutionContext) -> countDownAndWait(countDownLatch));

    SpawnActionContext dynamicSpawnStrategy = createSpawnStrategy(localStrategy, remoteStrategy);

    Spawn spawn = newDynamicSpawn();
    dynamicSpawnStrategy.exec(spawn, actionExecutionContext);

    assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(localStrategy.succeeded()).isFalse();
    assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(remoteStrategy.succeeded()).isTrue();

    assertThat(outErr.outAsLatin1()).contains("output files written with MockRemoteSpawnStrategy");
    assertThat(outErr.outAsLatin1()).doesNotContain("MockLocalSpawnStrategy");
  }

  @Test
  public void actionFailsIfLocalFailsImmediatelyEvenIfRemoteSucceedsLater() throws Exception {
    CountDownLatch countDownLatch = new CountDownLatch(2);

    MockLocalSpawnStrategy localStrategy =
        new MockLocalSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              countDownAndWait(countDownLatch);
              self.failExecution(actionExecutionContext);
            });

    MockRemoteSpawnStrategy remoteStrategy =
        new MockRemoteSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              countDownAndWait(countDownLatch);
              Thread.sleep(2000);
            });

    SpawnActionContext dynamicSpawnStrategy = createSpawnStrategy(localStrategy, remoteStrategy);

    Spawn spawn = newDynamicSpawn();
    ExecException e =
        assertThrows(
            ExecException.class, () -> dynamicSpawnStrategy.exec(spawn, actionExecutionContext));
    assertThat(e).hasMessageThat().matches("MockLocalSpawnStrategy failed to execute the Spawn");

    assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(localStrategy.succeeded()).isFalse();
    assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(remoteStrategy.succeeded()).isFalse();

    assertThat(outErr.outAsLatin1()).contains("action failed with MockLocalSpawnStrategy");
    assertThat(outErr.outAsLatin1()).doesNotContain("MockRemoteSpawnStrategy");
  }

  @Test
  public void actionFailsIfRemoteFailsImmediatelyEvenIfLocalSucceedsLater() throws Exception {
    CountDownLatch countDownLatch = new CountDownLatch(2);

    MockLocalSpawnStrategy localStrategy =
        new MockLocalSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              countDownAndWait(countDownLatch);
              Thread.sleep(2000);
            });

    MockRemoteSpawnStrategy remoteStrategy =
        new MockRemoteSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              countDownAndWait(countDownLatch);
              self.failExecution(actionExecutionContext);
            });

    SpawnActionContext dynamicSpawnStrategy = createSpawnStrategy(localStrategy, remoteStrategy);

    Spawn spawn = newDynamicSpawn();
    ExecException e =
        assertThrows(
            ExecException.class, () -> dynamicSpawnStrategy.exec(spawn, actionExecutionContext));
    assertThat(e).hasMessageThat().matches("MockRemoteSpawnStrategy failed to execute the Spawn");

    assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(localStrategy.succeeded()).isFalse();
    assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(remoteStrategy.succeeded()).isFalse();

    assertThat(outErr.outAsLatin1()).contains("action failed with MockRemoteSpawnStrategy");
    assertThat(outErr.outAsLatin1()).doesNotContain("MockLocalSpawnStrategy");
  }

  @Test
  public void actionFailsIfLocalAndRemoteFail() throws Exception {
    CountDownLatch countDownLatch = new CountDownLatch(2);

    MockLocalSpawnStrategy localStrategy =
        new MockLocalSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              countDownAndWait(countDownLatch);
              self.failExecution(actionExecutionContext);
            });

    MockRemoteSpawnStrategy remoteStrategy =
        new MockRemoteSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              countDownAndWait(countDownLatch);
              self.failExecution(actionExecutionContext);
            });

    SpawnActionContext dynamicSpawnStrategy = createSpawnStrategy(localStrategy, remoteStrategy);

    Spawn spawn = newDynamicSpawn();
    ExecException e =
        assertThrows(
            ExecException.class, () -> dynamicSpawnStrategy.exec(spawn, actionExecutionContext));
    assertThat(e)
        .hasMessageThat()
        .matches("Mock(Local|Remote)SpawnStrategy failed to execute the Spawn");

    assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(localStrategy.succeeded()).isFalse();
    assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(remoteStrategy.succeeded()).isFalse();
  }

  @Test
  public void stopConcurrentSpawnsWaitForCompletion() throws Exception {
    if (legacyBehavior) {
      // The legacy spawn scheduler does not implement cross-cancellations of the two parallel
      // branches so this test makes no sense in that case.
      Logger.getLogger(DynamicSpawnStrategyTest.class.getName()).info("Skipping test");
      return;
    }

    CountDownLatch countDownLatch = new CountDownLatch(2);

    AtomicBoolean slowCleanupFinished = new AtomicBoolean(false);
    MockLocalSpawnStrategy localStrategy =
        new MockLocalSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              try {
                countDownAndWait(countDownLatch);
                // Block indefinitely waiting for the remote branch to interrupt us.
                Thread.sleep(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
                fail("Should have been interrupted");
              } catch (InterruptedException e) {
                // Wait for "long enough" hoping that the remoteStrategy will have enough time to
                // check the value of slowCleanupFinished before we finish this sleep, in case we
                // have a bug.
                Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
                slowCleanupFinished.set(true);
              }
            });

    MockRemoteSpawnStrategy remoteStrategy =
        new MockRemoteSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> countDownAndWait(countDownLatch),
            (self, spawn, actionExecutionContext) -> {
              // This runs after we have asked the local spawn to complete and, in theory, awaited
              // for InterruptedException to propagate. Make sure that's the case here by checking
              // that we did indeed wait for the slow process.
              if (!slowCleanupFinished.get()) {
                fail("Did not await for the other branch to do its cleanup");
              }
            });

    SpawnActionContext dynamicSpawnStrategy = createSpawnStrategy(localStrategy, remoteStrategy);

    Spawn spawn = newDynamicSpawn();
    dynamicSpawnStrategy.exec(spawn, actionExecutionContext);

    assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(localStrategy.succeeded()).isFalse();
    assertThat(remoteStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(remoteStrategy.succeeded()).isTrue();
  }

  @Test
  public void noDeadlockWithSingleThreadedExecutor() throws Exception {
    MockLocalSpawnStrategy localStrategy = new MockLocalSpawnStrategy(testRoot);
    MockRemoteSpawnStrategy remoteStrategy = new MockRemoteSpawnStrategy(testRoot);
    SpawnActionContext dynamicSpawnStrategy =
        createSpawnStrategyWithExecutor(
            localStrategy, remoteStrategy, Executors.newSingleThreadExecutor());

    Spawn spawn = newDynamicSpawn();
    dynamicSpawnStrategy.exec(spawn, actionExecutionContext);

    assertThat(localStrategy.getExecutedSpawn()).isEqualTo(spawn);
    assertThat(localStrategy.succeeded()).isTrue();

    /**
     * The single-threaded executorService#invokeAny does not comply to the contract where
     * the callables are *always* called sequentially. In this case, both spawns will start
     * executing, but the local one will always succeed as it's the first to be called. The remote
     * one will then be cancelled, or is null if the local one completes before the remote one
     * starts.
     *
     * See the documentation of {@link BoundedExectorService#invokeAny(Collection)}, specifically:
     * "The following is less efficient (it goes on submitting tasks even if there is some task
     * already finished), but quite straight-forward.".
     */
    assertThat(remoteStrategy.getExecutedSpawn()).isAnyOf(spawn, null);
    assertThat(remoteStrategy.succeeded()).isFalse();
  }

  @Test
  public void interruptDuringExecutionDoesActuallyInterruptTheExecution() throws Exception {
    CountDownLatch countDownLatch = new CountDownLatch(2);

    MockLocalSpawnStrategy localStrategy =
        new MockLocalSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              countDownAndWait(countDownLatch);
              Thread.sleep(60000);
            });

    MockRemoteSpawnStrategy remoteStrategy =
        new MockRemoteSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              countDownAndWait(countDownLatch);
              Thread.sleep(60000);
            });

    SpawnActionContext dynamicSpawnStrategy = createSpawnStrategy(localStrategy, remoteStrategy);

    TestThread testThread =
        new TestThread(
            () -> {
              try {
                Spawn spawn = newDynamicSpawn();
                dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
              } catch (InterruptedException e) {
                // This is expected.
              }
            });
    testThread.start();
    countDownLatch.await(5, TimeUnit.SECONDS);
    testThread.interrupt();
    testThread.joinAndAssertState(5000);

    assertThat(outErr.getOutputPath().exists()).isFalse();
    assertThat(outErr.getErrorPath().exists()).isFalse();
  }

  /** Hook to validate the result of the strategy's execution. */
  @FunctionalInterface
  interface CheckExecResult {
    void check(@Nullable Exception e) throws Exception;
  }

  /**
   * Runs a test to check that both spawns finished under various conditions before the strategy's
   * {@code exec} method returns control.
   *
   * @param executionFails causes one of the branches in the execution to terminate with an
   *     execution exception
   * @param interruptThread causes the strategy's execution to be interrupted while it is waiting
   *     for its branches to complete
   * @param checkExecResult a lambda to validate the result of the execution. Receives null if the
   *     execution completed successfully, or else the raised exception.
   */
  private void assertThatStrategyWaitsForBothSpawnsToFinish(
      boolean executionFails, boolean interruptThread, CheckExecResult checkExecResult)
      throws Exception {
    if (!legacyBehavior) {
      // TODO(jmmv): I've spent *days* trying to make these tests work reliably with the new dynamic
      // spawn scheduler implementation but I keep encountering tricky race conditions everywhere. I
      // have strong reasons to believe that the races are due to inherent problems in these tests,
      // not in the actual DynamicSpawnScheduler implementation. So whatever. I'll revisit these
      // later as a new set of tests once I'm less tired^W^W^W the legacy spawn scheduler goes away.
      Logger.getLogger(DynamicSpawnStrategyTest.class.getName()).info("Skipping test");
      return;
    }

    AtomicBoolean stopLocal = new AtomicBoolean(false);
    CountDownLatch executionCanProceed = new CountDownLatch(2);
    CountDownLatch remoteDone = new CountDownLatch(1);

    MockLocalSpawnStrategy localStrategy =
        new MockLocalSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              executionCanProceed.countDown();

              // We cannot use a synchronization primitive to block termination of this thread
              // because we expect to be interrupted by the remote strategy, and even in that case
              // we want to control exactly when this finishes. We could wait for and swallow the
              // interrupt before waiting again on a latch here... but swallowing the interrupt can
              // lead to race conditions.
              while (!stopLocal.get()) {
                Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MILLISECONDS);
              }
              throw new InterruptedException("Local stopped");
            });

    MockRemoteSpawnStrategy remoteStrategy =
        new MockRemoteSpawnStrategy(
            testRoot,
            (self, spawn, actionExecutionContext) -> {
              try {
                // Wait until the local branch has started so that our completion causes it to be
                // interrupted in a known location.
                countDownAndWait(executionCanProceed);

                if (executionFails) {
                  self.failExecution(actionExecutionContext);
                  throw new AssertionError("Not reachable");
                }
              } finally {
                remoteDone.countDown();
              }
            });

    SpawnActionContext dynamicSpawnStrategy = createSpawnStrategy(localStrategy, remoteStrategy);
    TestThread testThread =
        new TestThread(
            () -> {
              try {
                Spawn spawn = newDynamicSpawn();
                dynamicSpawnStrategy.exec(spawn, actionExecutionContext);
                checkExecResult.check(null);
              } catch (Exception e) {
                checkExecResult.check(e);
              }
            });
    testThread.start();
    try {
      remoteDone.await();
      // At this point, the remote branch is done and the local branch is waiting until we allow it
      // to complete later on. This is necessary to let us assert the state of the thread's
      // liveliness.
      //
      // However, note that "done" just means that our DoExec hook for remoteStrategy finished.
      // Any exception raised from within it may still be propagating up, so the interrupt below
      // races with that (and thus an InterruptedException can "win" over our own exception). There
      // is no way to handle this condition in the test other than having to acknowledge that it may
      // happen.

      if (interruptThread) {
        testThread.interrupt();
      }

      // The thread running the exec via the strategy must still be alive regardless of our
      // interrupt request (because the local branch is stuck). Wait for a little bit to ensure
      // this is true; any multi-second wait should be sufficient to catch the majority of the
      // bugs.
      testThread.join(2000);
      assertThat(testThread.isAlive()).isTrue();
    } finally {
      // Unblocking the local branch allows the strategy to collect its result and then unblock the
      // thread.
      stopLocal.set(true);
      testThread.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
    }
  }

  @Test
  public void strategyWaitsForBothSpawnsToFinish() throws Exception {
    assertThatStrategyWaitsForBothSpawnsToFinish(
        /*executionFails=*/ false,
        /*interruptThread=*/ false,
        (e) -> {
          if (e != null) {
            throw new IllegalStateException("Expected exec to finish successfully", e);
          }
        });
  }

  @Test
  public void strategyWaitsForBothSpawnsToFinishEvenIfInterrupted() throws Exception {
    assertThatStrategyWaitsForBothSpawnsToFinish(
        /*executionFails=*/ false,
        /*interruptThread=*/ true,
        (e) -> {
          if (e == null) {
            fail("No exception raised");
          } else if (e instanceof InterruptedException) {
            assertThat(Thread.currentThread().isInterrupted()).isFalse();
          } else {
            throw e;
          }
        });
  }

  @Test
  public void strategyWaitsForBothSpawnsToFinishOnFailure() throws Exception {
    assertThatStrategyWaitsForBothSpawnsToFinish(
        /*executionFails=*/ true,
        /*interruptThread=*/ false,
        (e) -> {
          if (e == null) {
            fail("No exception raised");
          } else if (e instanceof ExecException) {
            assertThat(Thread.currentThread().isInterrupted()).isFalse();
          } else {
            throw e;
          }
        });
  }

  @Test
  public void strategyWaitsForBothSpawnsToFinishOnFailureEvenIfInterrupted() throws Exception {
    assertThatStrategyWaitsForBothSpawnsToFinish(
        /*executionFails=*/ true,
        /*interruptThread=*/ true,
        (e) -> {
          if (e == null) {
            fail("No exception raised");
          } else if (e instanceof InterruptedException) {
            // See comment in strategyWaitsForBothSpawnsToFinish regarding the race between the
            // exception we raise on failure and the interrupt. We have to handle this case even
            // though it is supposedly rare.
          } else if (e instanceof ExecException) {
            assertThat(Thread.currentThread().isInterrupted()).isTrue();
          } else {
            throw e;
          }
        });
  }

  private void assertThatStrategyPropagatesException(
      DoExec localExec, DoExec remoteExec, Exception expectedException) throws Exception {
    checkArgument(
        !(expectedException instanceof IllegalStateException),
        "Using an IllegalStateException for testing is fragile because we use that exception "
            + "internally in the DynamicSpawnScheduler and we cannot distinguish it from the "
            + "test's own exception");

    MockLocalSpawnStrategy localStrategy = new MockLocalSpawnStrategy(testRoot, localExec);
    MockRemoteSpawnStrategy remoteStrategy = new MockRemoteSpawnStrategy(testRoot, remoteExec);
    SpawnActionContext dynamicSpawnStrategy = createSpawnStrategy(localStrategy, remoteStrategy);

    Spawn spawn = newDynamicSpawn();
    Exception e =
        assertThrows(
            expectedException.getClass(),
            () -> dynamicSpawnStrategy.exec(spawn, actionExecutionContext));
    assertThat(e).hasMessageThat().matches(expectedException.getMessage());

    Spawn executedSpawn = localStrategy.getExecutedSpawn();
    executedSpawn = executedSpawn == null ? remoteStrategy.getExecutedSpawn() : executedSpawn;
    assertThat(executedSpawn).isEqualTo(spawn);
    assertThat(localStrategy.succeeded()).isFalse();
    assertThat(remoteStrategy.succeeded()).isFalse();
  }

  @Test
  public void strategyPropagatesFasterLocalException() throws Exception {
    RuntimeException e = new IllegalArgumentException("Local spawn execution exception");
    DoExec localExec =
        (self, spawn, actionExecutionContext) -> {
          throw e;
        };

    DoExec remoteExec =
        (self, spawn, actionExecutionContext) -> {
          Thread.sleep(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
          throw new AssertionError("Not reachable");
        };

    assertThatStrategyPropagatesException(
        localExec, remoteExec, legacyBehavior ? new UserExecException(e) : e);
  }

  @Test
  public void strategyPropagatesFasterRemoteException() throws Exception {
    DoExec localExec =
        (self, spawn, actionExecutionContext) -> {
          Thread.sleep(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
          throw new AssertionError("Not reachable");
        };

    RuntimeException e = new IllegalArgumentException("Remote spawn execution exception");
    DoExec remoteExec =
        (self, spawn, actionExecutionContext) -> {
          throw e;
        };

    assertThatStrategyPropagatesException(
        localExec, remoteExec, legacyBehavior ? new UserExecException(e) : e);
  }
}