// Copyright 2022 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.remote;

import static com.google.common.collect.Iterables.getOnlyElement;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static com.google.devtools.build.lib.vfs.FileSystemUtils.writeContent;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertThrows;
import static org.junit.Assume.assumeFalse;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.eventbus.Subscribe;
import com.google.devtools.build.lib.actions.ActionExecutedEvent;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.BuildFailedException;
import com.google.devtools.build.lib.actions.CachedActionEvent;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase;
import com.google.devtools.build.lib.skyframe.ActionExecutionValue;
import com.google.devtools.build.lib.skyframe.TreeArtifactValue;
import com.google.devtools.build.lib.util.OS;
import com.google.devtools.build.lib.util.io.RecordingOutErr;
import com.google.devtools.build.lib.vfs.Path;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.junit.Test;

/** Base class for integration tests for BwoB. */
public abstract class BuildWithoutTheBytesIntegrationTestBase extends BuildIntegrationTestCase {
  // Concrete implementations should by default set the necessary flags to download minimal outputs.
  // These methods should override the necessary flags to download top-level outputs or all outputs.
  protected abstract void setDownloadToplevel();

  protected abstract void setDownloadAll();

  protected abstract void assertOutputEquals(Path path, String expectedContent) throws Exception;

  protected abstract void assertOutputContains(String content, String contains) throws Exception;

  protected abstract void evictAllBlobs() throws Exception;

  protected abstract boolean hasAccessToRemoteOutputs();

  protected abstract void injectFile(byte[] content);

  protected void waitDownloads() throws Exception {
    // Trigger afterCommand of modules so that downloads are waited.
    runtimeWrapper.newCommand();
  }

  // TODO(b/281655526) incompatible with Skymeld.
  protected void setIncompatibleWithSkymeld() {
    addOptions("--noexperimental_merged_skyframe_analysis_execution");
  }

  @Test
  public void outputsAreNotDownloaded() throws Exception {
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo foo > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        ")");

    buildTarget("//:foobar");
    waitDownloads();

    assertOutputsDoNotExist("//:foo");
    assertOutputsDoNotExist("//:foobar");
  }

  @Test
  public void disableRunfiles_buildSuccessfully() throws Exception {
    // Disable on Windows since it fails for unknown reasons.
    // TODO(chiwang): Enable it on windows.
    if (OS.getCurrent() == OS.WINDOWS) {
      return;
    }
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  cmd = 'echo foo > $@',",
        "  outs = ['foo.data'],",
        ")",
        "sh_test(",
        "  name = 'foobar',",
        "  srcs = ['test.sh'],",
        "  data = [':foo'],",
        ")");
    write("test.sh");
    getWorkspace().getRelative("test.sh").setExecutable(true);
    addOptions("--build_runfile_links", "--enable_runfiles=no");

    buildTarget("//:foobar");
  }

  @Test
  public void downloadOutputsWithRegex() throws Exception {
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo foo > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        ")");
    addOptions("--experimental_remote_download_regex=.*foo\\.txt$");

    buildTarget("//:foobar");
    waitDownloads();

    assertValidOutputFile("out/foo.txt", "foo\n");
    assertOutputsDoNotExist("//:foobar");

    // Assert that no actions have been executed for the next incremental build since nothing
    // changed
    ActionEventCollector actionEventCollector = new ActionEventCollector();
    getRuntimeWrapper().registerSubscriber(actionEventCollector);
    // Override out/foo.txt with the same content
    {
      var path = getOutputPath("out/foo.txt");
      var isWritable = path.isWritable();
      if (!isWritable) {
        path.setWritable(true);
      }
      writeContent(path, UTF_8, "foo\n");
      if (!isWritable) {
        path.setWritable(false);
      }
    }
    buildTarget("//:foobar");
    assertThat(actionEventCollector.getActionExecutedEvents()).isEmpty();
  }

  @Test
  public void downloadOutputsWithRegex_deleteOutput_reDownload() throws Exception {
    // Arrange: Do a clean build and download out/foo.txt
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo foo > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        ")");
    addOptions("--experimental_remote_download_regex=.*foo\\.txt$");

    buildTarget("//:foobar");
    waitDownloads();

    assertValidOutputFile("out/foo.txt", "foo\n");
    assertOutputsDoNotExist("//:foobar");

    // Arrange: Delete out/foo.txt and do an incremental build
    getOutputPath("out/foo.txt").delete();
    ActionEventCollector actionEventCollector = new ActionEventCollector();
    getRuntimeWrapper().registerSubscriber(actionEventCollector);
    buildTarget("//:foobar");
    waitDownloads();

    // Assert: out/foo.txt is re-downloaded
    assertThat(actionEventCollector.getActionExecutedEvents()).hasSize(1);
    assertValidOutputFile("out/foo.txt", "foo\n");
  }

  @Test
  public void downloadOutputsWithRegex_changeRegex_downloadNewMatches() throws Exception {
    // Arrange: Do a clean build
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo foo > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        ")");

    buildTarget("//:foobar");
    // Add the new option here because waitDownloads below will internally create a new command
    // which will parse the new option.
    addOptions("--experimental_remote_download_regex=.*foobar\\.txt$");
    waitDownloads();

    assertOutputsDoNotExist("//:foo");
    assertOutputsDoNotExist("//:foobar");

    // Arrange: Change regex
    ActionEventCollector actionEventCollector = new ActionEventCollector();
    getRuntimeWrapper().registerSubscriber(actionEventCollector);
    buildTarget("//:foobar");
    waitDownloads();

    // Assert: out/foobar.txt is downloaded
    assertThat(actionEventCollector.getActionExecutedEvents()).hasSize(1);
    assertValidOutputFile("out/foobar.txt", "foo\nbar\n");
  }

  @Test
  public void downloadOutputsWithRegex_treeOutput_regexMatchesTreeFile() throws Exception {
    // Disable on Windows since it fails for unknown reasons.
    // TODO(chiwang): Enable it on windows.
    if (OS.getCurrent() == OS.WINDOWS) {
      return;
    }
    writeOutputDirRule();
    write(
        "BUILD",
        "load(':output_dir.bzl', 'output_dir')",
        "output_dir(",
        "  name = 'foo',",
        "  content_map = {'file-1': '1', 'file-2': '2', 'file-3': '3'},",
        ")");
    addOptions("--experimental_remote_download_regex=.*foo/file-2$");

    buildTarget("//:foo");
    waitDownloads();

    assertValidOutputFile("foo/file-2", "2");
    assertOutputDoesNotExist("foo/file-1");
    assertOutputDoesNotExist("foo/file-3");
  }

  @Test
  public void downloadOutputsWithRegex_treeOutput_regexMatchesTreeRoot() throws Exception {
    writeOutputDirRule();
    write(
        "BUILD",
        "load(':output_dir.bzl', 'output_dir')",
        "output_dir(",
        "  name = 'foo',",
        "  content_map = {'file-1': '1', 'file-2': '2', 'file-3': '3'},",
        ")");
    addOptions("--experimental_remote_download_regex=.*foo$");

    buildTarget("//:foo");
    waitDownloads();

    assertThat(getOutputPath("foo").exists()).isTrue();
    assertOutputDoesNotExist("foo/file-1");
    assertOutputDoesNotExist("foo/file-2");
    assertOutputDoesNotExist("foo/file-3");
  }

  @Test
  public void downloadOutputsWithRegex_regexMatchParentPath_filesNotDownloaded() throws Exception {
    write(
        "BUILD",
        "genrule(",
        "  name = 'file-1',",
        "  srcs = [],",
        "  outs = ['foo/file-1'],",
        "  cmd = 'echo file-1 > $@',",
        ")",
        "genrule(",
        "  name = 'file-2',",
        "  srcs = [],",
        "  outs = ['foo/file-2'],",
        "  cmd = 'echo file-2 > $@',",
        ")",
        "genrule(",
        "  name = 'file-3',",
        "  srcs = [],",
        "  outs = ['foo/file-3'],",
        "  cmd = 'echo file-3 > $@',",
        ")");
    addOptions("--experimental_remote_download_regex=.*foo$");

    buildTarget("//:file-1", "//:file-2", "//:file-3");
    waitDownloads();

    assertOutputDoesNotExist("foo/file-1");
    assertOutputDoesNotExist("foo/file-2");
    assertOutputDoesNotExist("foo/file-3");
  }

  @Test
  public void intermediateOutputsAreInputForLocalActions_prefetchIntermediateOutputs()
      throws Exception {
    // Test that a remote-only output that's an input to a local action is downloaded lazily before
    // executing the local action.
    write(
        "a/BUILD",
        "genrule(",
        "  name = 'remote',",
        "  srcs = [],",
        "  outs = ['remote.txt'],",
        "  cmd = 'echo -n remote > $@',",
        ")",
        "",
        "genrule(",
        "  name = 'local',",
        "  srcs = [':remote'],",
        "  outs = ['local.txt'],",
        "  cmd = 'cat $(location :remote) > $@ && echo -n local >> $@',",
        "  tags = ['no-remote'],",
        ")");

    buildTarget("//a:remote");
    waitDownloads();
    assertOutputsDoNotExist("//a:remote");
    buildTarget("//a:local");
    waitDownloads();

    assertOnlyOutputContent("//a:remote", "remote.txt", "remote");
    assertOnlyOutputContent("//a:local", "local.txt", "remotelocal");
  }

  @Test
  public void symlinkToSourceFile() throws Exception {
    write(
        "a/defs.bzl",
        "def _impl(ctx):",
        "  if ctx.attr.chain_length < 1:",
        "    fail('chain_length must be > 0')",
        "",
        "  file = ctx.file.target",
        "",
        "  for i in range(ctx.attr.chain_length):",
        "    sym = ctx.actions.declare_file(ctx.label.name + '.sym' + str(i))",
        "    ctx.actions.symlink(output = sym, target_file = file)",
        "    file = sym",
        "",
        "  out = ctx.actions.declare_file(ctx.label.name + '.out')",
        "  ctx.actions.run_shell(",
        "    inputs = [sym],",
        "    outputs = [out],",
        "    command = '[[ hello == $(cat $1) ]] && touch $2',",
        "    arguments = [sym.path, out.path],",
        "    execution_requirements = {'no-remote': ''} if ctx.attr.local else {},",
        "  )",
        "",
        "  return DefaultInfo(files = depset([out]))",
        "",
        "my_rule = rule(",
        "  implementation = _impl,",
        "  attrs = {",
        "    'target': attr.label(allow_single_file = True),",
        "    'chain_length': attr.int(),",
        "    'local': attr.bool(),",
        "  },",
        ")");

    write(
        "a/BUILD",
        "load(':defs.bzl', 'my_rule')",
        "",
        "my_rule(name = 'one_local', target = 'src.txt', local = True, chain_length = 1)",
        "my_rule(name = 'two_local', target = 'src.txt', local = True, chain_length = 2)",
        "my_rule(name = 'one_remote', target = 'src.txt', local = False, chain_length = 1)",
        "my_rule(name = 'two_remote', target = 'src.txt', local = False, chain_length = 2)");

    write("a/src.txt", "hello");

    buildTarget("//a:one_local", "//a:two_local", "//a:one_remote", "//a:two_remote");
  }

  @Test
  public void symlinkToGeneratedFile() throws Exception {
    injectFile("hello".getBytes(UTF_8));
    write(
        "a/defs.bzl",
        "def _impl(ctx):",
        "  if ctx.attr.chain_length < 1:",
        "    fail('chain_length must be > 0')",
        "",
        "  file = ctx.actions.declare_file(ctx.label.name + '.file')",
        // Use ctx.actions.run_shell instead of ctx.actions.write, so that it runs remotely.
        "  ctx.actions.run_shell(",
        "    outputs = [file],",
        "    command = 'echo -n hello > $1',",
        "    arguments = [file.path],",
        "  )",
        "",
        "  for i in range(ctx.attr.chain_length):",
        "    sym = ctx.actions.declare_file(ctx.label.name + '.sym' + str(i))",
        "    ctx.actions.symlink(output = sym, target_file = file)",
        "    file = sym",
        "",
        "  out = ctx.actions.declare_file(ctx.label.name + '.out')",
        "  ctx.actions.run_shell(",
        "    inputs = [sym],",
        "    outputs = [out],",
        "    command = '[[ hello == $(cat $1) ]] && touch $2',",
        "    arguments = [sym.path, out.path],",
        "    execution_requirements = {'no-remote': ''} if ctx.attr.local else {},",
        "  )",
        "",
        "  return DefaultInfo(files = depset([out]))",
        "",
        "my_rule = rule(",
        "  implementation = _impl,",
        "  attrs = {",
        "    'chain_length': attr.int(),",
        "    'local': attr.bool(),",
        "  },",
        ")");

    write(
        "a/BUILD",
        "load(':defs.bzl', 'my_rule')",
        "",
        "my_rule(name = 'one_local', local = True, chain_length = 1)",
        "my_rule(name = 'two_local', local = True, chain_length = 2)",
        "my_rule(name = 'one_remote', local = False, chain_length = 1)",
        "my_rule(name = 'two_remote', local = False, chain_length = 2)");

    buildTarget("//a:one_local", "//a:two_local", "//a:one_remote", "//a:two_remote");
  }

  @Test
  public void symlinkToDirectory() throws Exception {
    injectFile("hello".getBytes(UTF_8));
    write(
        "a/defs.bzl",
        "def _impl(ctx):",
        "  if ctx.attr.chain_length < 1:",
        "    fail('chain_length must be > 0')",
        "",
        "  dir = ctx.actions.declare_directory(ctx.label.name + '.dir')",
        "  ctx.actions.run_shell(",
        "    outputs = [dir],",
        "    command = 'mkdir -p $1/some/path && echo -n hello > $1/some/path/inside.txt',",
        "    arguments = [dir.path],",
        "  )",
        "",
        "  for i in range(ctx.attr.chain_length):",
        "    sym = ctx.actions.declare_directory(ctx.label.name + '.sym' + str(i))",
        "    ctx.actions.symlink(output = sym, target_file = dir)",
        "    dir = sym",
        "",
        "  out = ctx.actions.declare_file(ctx.label.name + '.out')",
        "  ctx.actions.run_shell(",
        "    inputs = [sym],",
        "    outputs = [out],",
        "    command = '[[ hello == $(cat $1/some/path/inside.txt) ]] && touch $2',",
        "    arguments = [sym.path, out.path],",
        "    execution_requirements = {'no-remote': ''} if ctx.attr.local else {},",
        "  )",
        "",
        "  return DefaultInfo(files = depset([out]))",
        "",
        "my_rule = rule(",
        "  implementation = _impl,",
        "  attrs = {",
        "    'chain_length': attr.int(),",
        "    'local': attr.bool()",
        "  },",
        ")");

    write(
        "a/BUILD",
        "load(':defs.bzl', 'my_rule')",
        "",
        "my_rule(name = 'one_local', local = True, chain_length = 1)",
        "my_rule(name = 'two_local', local = True, chain_length = 2)",
        "my_rule(name = 'one_remote', local = False, chain_length = 1)",
        "my_rule(name = 'two_remote', local = False, chain_length = 2)");

    buildTarget("//a:one_local", "//a:two_local", "//a:one_remote", "//a:two_remote");
  }

  @Test
  public void symlinkToNestedFile() throws Exception {
    injectFile("hello".getBytes(UTF_8));
    addOptions("--noincompatible_strict_conflict_checks");

    write(
        "a/defs.bzl",
        "def _impl(ctx):",
        "  if ctx.attr.chain_length < 1:",
        "    fail('chain_length must be > 0')",
        "",
        "  dir = ctx.actions.declare_directory(ctx.label.name + '.dir')",
        "  file = ctx.actions.declare_file(ctx.label.name + '.dir/some/path/inside.txt')",
        "  ctx.actions.run_shell(",
        "    outputs = [dir, file],",
        "    command = 'mkdir -p $1/some/path && echo -n hello > $1/some/path/inside.txt',",
        "    arguments = [dir.path],",
        "  )",
        "",
        "  for i in range(ctx.attr.chain_length):",
        "    sym = ctx.actions.declare_file(ctx.label.name + '.sym' + str(i))",
        "    ctx.actions.symlink(output = sym, target_file = file)",
        "    file = sym",
        "",
        "  out = ctx.actions.declare_file(ctx.label.name + '.out')",
        "  ctx.actions.run_shell(",
        "    inputs = [sym],",
        "    outputs = [out],",
        "    command = '[[ hello == $(cat $1) ]] && touch $2',",
        "    arguments = [sym.path, out.path],",
        "    execution_requirements = {'no-remote': ''} if ctx.attr.local else {},",
        "  )",
        "",
        "  return DefaultInfo(files = depset([out]))",
        "",
        "my_rule = rule(",
        "  implementation = _impl,",
        "  attrs = {",
        "    'chain_length': attr.int(),",
        "    'local': attr.bool(),",
        "  },",
        ")");

    write(
        "a/BUILD",
        "load(':defs.bzl', 'my_rule')",
        "",
        "my_rule(name = 'one_local', local = True, chain_length = 1)",
        "my_rule(name = 'two_local', local = True, chain_length = 2)",
        "my_rule(name = 'one_remote', local = False, chain_length = 1)",
        "my_rule(name = 'two_remote', local = False, chain_length = 2)");

    buildTarget("//a:one_local", "//a:two_local", "//a:one_remote", "//a:two_remote");
  }

  @Test
  public void symlinkToNestedDirectory() throws Exception {
    injectFile("hello".getBytes(UTF_8));
    addOptions("--noincompatible_strict_conflict_checks");

    write(
        "a/defs.bzl",
        "def _impl(ctx):",
        "  if ctx.attr.chain_length < 1:",
        "    fail('chain_length must be > 0')",
        "",
        "  dir = ctx.actions.declare_directory(ctx.label.name + '.dir')",
        "  subdir = ctx.actions.declare_directory(ctx.label.name + '.dir/some/path')",
        "  ctx.actions.run_shell(",
        "    outputs = [dir, subdir],",
        "    command = 'mkdir -p $1/some/path && echo -n hello > $1/some/path/inside.txt',",
        "    arguments = [dir.path],",
        "  )",
        "",
        "  for i in range(ctx.attr.chain_length):",
        "    sym = ctx.actions.declare_directory(ctx.label.name + '.sym' + str(i))",
        "    ctx.actions.symlink(output = sym, target_file = subdir)",
        "    subdir = sym",
        "",
        "  out = ctx.actions.declare_file(ctx.label.name + '.out')",
        "  ctx.actions.run_shell(",
        "    inputs = [sym],",
        "    outputs = [out],",
        "    command = '[[ hello == $(cat $1/inside.txt) ]] && touch $2',",
        "    arguments = [sym.path, out.path],",
        "    execution_requirements = {'no-remote': ''} if ctx.attr.local else {},",
        "  )",
        "",
        "  return DefaultInfo(files = depset([out]))",
        "",
        "my_rule = rule(",
        "  implementation = _impl,",
        "  attrs = {",
        "    'chain_length': attr.int(),",
        "    'local': attr.bool(),",
        "  },",
        ")");

    write(
        "a/BUILD",
        "load(':defs.bzl', 'my_rule')",
        "",
        "my_rule(name = 'one_local', local = True, chain_length = 1)",
        "my_rule(name = 'two_local', local = True, chain_length = 2)",
        "my_rule(name = 'one_remote', local = False, chain_length = 1)",
        "my_rule(name = 'two_remote', local = False, chain_length = 2)");

    buildTarget("//a:one_local", "//a:two_local", "//a:one_remote", "//a:two_remote");
  }

  @Test
  public void localAction_stdoutIsReported() throws Exception {
    // Disable on Windows since it fails for unknown reasons.
    // TODO(chiwang): Enable it on windows.
    if (OS.getCurrent() == OS.WINDOWS) {
      return;
    }
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo my-output-message > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) && touch $@',",
        "  tags = ['no-remote'],",
        ")");
    RecordingOutErr outErr = new RecordingOutErr();
    this.outErr = outErr;

    buildTarget("//:foobar");
    waitDownloads();

    assertOutputContains(outErr.outAsLatin1(), "my-output-message");
  }

  @Test
  public void localAction_stderrIsReported() throws Exception {
    // Disable on Windows since it fails for unknown reasons.
    // TODO(chiwang): Enable it on windows.
    if (OS.getCurrent() == OS.WINDOWS) {
      return;
    }
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo my-error-message > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) >&2 && exit 1',",
        "  tags = ['no-remote'],",
        ")");
    RecordingOutErr outErr = new RecordingOutErr();
    this.outErr = outErr;

    assertThrows(BuildFailedException.class, () -> buildTarget("//:foobar"));

    assertOutputContains(outErr.errAsLatin1(), "my-error-message");
  }

  @Test
  public void dynamicExecution_stdoutIsReported() throws Exception {
    // Disable on Windows since it fails for unknown reasons.
    // TODO(chiwang): Enable it on windows.
    if (OS.getCurrent() == OS.WINDOWS) {
      return;
    }
    addOptions("--internal_spawn_scheduler");
    addOptions("--strategy=Genrule=dynamic");
    addOptions("--experimental_local_execution_delay=9999999");
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo my-output-message > $@',",
        "  tags = ['no-local'],",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) && touch $@',",
        ")");
    RecordingOutErr outErr = new RecordingOutErr();
    this.outErr = outErr;

    buildTarget("//:foobar");
    waitDownloads();

    assertOutputContains(outErr.outAsLatin1(), "my-output-message");
  }

  @Test
  public void dynamicExecution_stderrIsReported() throws Exception {
    // Disable on Windows since it fails for unknown reasons.
    // TODO(chiwang): Enable it on windows.
    if (OS.getCurrent() == OS.WINDOWS) {
      return;
    }
    addOptions("--internal_spawn_scheduler");
    addOptions("--strategy=Genrule=dynamic");
    addOptions("--experimental_local_execution_delay=9999999");
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo my-error-message > $@',",
        "  tags = ['no-local'],",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) >&2 && exit 1',",
        ")");
    RecordingOutErr outErr = new RecordingOutErr();
    this.outErr = outErr;

    assertThrows(BuildFailedException.class, () -> buildTarget("//:foobar"));

    assertOutputContains(outErr.errAsLatin1(), "my-error-message");
  }

  @Test
  public void downloadToplevel_outputsFromAspect_notAggregated() throws Exception {
    setDownloadToplevel();
    writeCopyAspectRule(/* aggregate= */ false);
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = ['foo.in'],",
        "  outs = ['foo.out'],",
        "  cmd = 'cat $(SRCS) > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['foobar.out'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        ")");
    write("foo.in", "foo");

    addOptions("--aspects=rules.bzl%copy_aspect", "--output_groups=+copy");
    buildTarget("//:foobar");
    waitDownloads();

    assertValidOutputFile("foobar.out", "foo" + lineSeparator() + "bar\n");
    assertOutputDoesNotExist("foo.in.copy");
    assertValidOutputFile("foo.out.copy", "foo" + lineSeparator());
  }

  @Test
  public void downloadToplevel_outputsFromAspect_aggregated() throws Exception {
    setDownloadToplevel();
    writeCopyAspectRule(/* aggregate= */ true);
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = ['foo.in'],",
        "  outs = ['foo.out'],",
        "  cmd = 'cat $(SRCS) > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['foobar.out'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        ")");
    write("foo.in", "foo");

    addOptions("--aspects=rules.bzl%copy_aspect", "--output_groups=+copy");
    buildTarget("//:foobar");
    waitDownloads();

    assertValidOutputFile("foobar.out", "foo" + lineSeparator() + "bar\n");
    assertValidOutputFile("foo.in.copy", "foo" + lineSeparator());
    assertValidOutputFile("foo.out.copy", "foo" + lineSeparator());
  }

  @Test
  public void downloadToplevel_outputsFromAspect_notDownloadedIfNoOutputGroups() throws Exception {
    setDownloadToplevel();
    writeCopyAspectRule(/* aggregate= */ true);
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = ['foo.in'],",
        "  outs = ['foo.out'],",
        "  cmd = 'cat $(SRCS) > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['foobar.out'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        ")");
    write("foo.in", "foo");

    addOptions("--aspects=rules.bzl%copy_aspect");
    buildTarget("//:foobar");
    waitDownloads();

    assertValidOutputFile("foobar.out", "foo" + lineSeparator() + "bar\n");
    assertOutputDoesNotExist("foo.in.copy");
    assertOutputDoesNotExist("foo.out.copy");
  }

  @Test
  public void downloadToplevel_outputsFromImportantOutputGroupAreDownloaded() throws Exception {
    setDownloadToplevel();
    write(
        "rules.bzl",
        "def _gen_impl(ctx):",
        "  output = ctx.actions.declare_file(ctx.attr.name)",
        "  ctx.actions.run_shell(",
        "    outputs = [output],",
        "    arguments = [ctx.attr.content, output.path],",
        "    command = 'echo $1 > $2',",
        "  )",
        "  extra1 = ctx.actions.declare_file(ctx.attr.name + '1')",
        "  ctx.actions.run_shell(",
        "    outputs = [extra1],",
        "    arguments = [ctx.attr.content, extra1.path],",
        "    command = 'echo $1 > $2',",
        "  )",
        "  extra2 = ctx.actions.declare_file(ctx.attr.name + '2')",
        "  ctx.actions.run_shell(",
        "    outputs = [extra2],",
        "    arguments = [ctx.attr.content, extra2.path],",
        "    command = 'echo $1 > $2',",
        "  )",
        "  return [",
        "    DefaultInfo(files = depset([output])),",
        "    OutputGroupInfo(",
        "      extra1_files = depset([extra1]),",
        "      extra2_files = depset([extra2]),",
        "    ),",
        "  ]",
        "",
        "gen = rule(",
        "  implementation = _gen_impl,",
        "  attrs = {",
        "    'content': attr.string(mandatory = True),",
        "  }",
        ")");
    write(
        "BUILD",
        "load(':rules.bzl', 'gen')",
        "gen(",
        "  name = 'foo',",
        "  content = 'foo-content',",
        ")");
    addOptions("--output_groups=+extra1_files");

    buildTarget("//:foo");
    waitDownloads();

    assertValidOutputFile("foo", "foo-content\n");
    assertValidOutputFile("foo1", "foo-content\n");
    assertOutputDoesNotExist("foo2");
  }

  @Test
  public void downloadToplevel_outputsFromHiddenOutputGroupAreNotDownloaded() throws Exception {
    setDownloadToplevel();
    write(
        "rules.bzl",
        "def _gen_impl(ctx):",
        "  output = ctx.actions.declare_file(ctx.attr.name)",
        "  ctx.actions.run_shell(",
        "    outputs = [output],",
        "    arguments = [ctx.attr.content, output.path],",
        "    command = 'echo $1 > $2',",
        "  )",
        "  validation_file = ctx.actions.declare_file(ctx.attr.name + '.validation')",
        "  ctx.actions.run_shell(",
        "    outputs = [validation_file],",
        "    arguments = [ctx.attr.content, validation_file.path],",
        "    command = 'echo $1 > $2',",
        "  )",
        "  return [",
        "    DefaultInfo(files = depset([output])),",
        "    OutputGroupInfo(",
        "      _validation = depset([validation_file]),",
        "    ),",
        "  ]",
        "",
        "gen = rule(",
        "  implementation = _gen_impl,",
        "  attrs = {",
        "    'content': attr.string(mandatory = True),",
        "  }",
        ")");
    write(
        "BUILD",
        "load(':rules.bzl', 'gen')",
        "gen(",
        "  name = 'foo',",
        "  content = 'foo-content',",
        ")");
    addOptions("--output_groups=+_validation");

    buildTarget("//:foo");
    waitDownloads();

    assertValidOutputFile("foo", "foo-content\n");
    assertOutputDoesNotExist("foo.validation");
  }

  @Test
  public void downloadToplevel_treeArtifacts() throws Exception {
    // Disable on Windows since it fails for unknown reasons.
    // TODO(chiwang): Enable it on windows.
    if (OS.getCurrent() == OS.WINDOWS) {
      return;
    }
    setDownloadToplevel();
    writeOutputDirRule();
    write(
        "BUILD",
        "load(':output_dir.bzl', 'output_dir')",
        "output_dir(",
        "  name = 'foo',",
        "  content_map = {'file-1': '1', 'file-2': '2', 'file-3': '3'},",
        ")");

    buildTarget("//:foo");

    assertValidOutputFile("foo/file-1", "1");
    assertValidOutputFile("foo/file-2", "2");
    assertValidOutputFile("foo/file-3", "3");
    // TODO(chiwang): Make metadata for downloaded outputs local.
    // assertThat(getMetadata("//:foo").values().stream().noneMatch(FileArtifactValue::isRemote))
    //     .isTrue();
  }

  @Test
  public void downloadToplevel_multipleToplevelTargets() throws Exception {
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo1',",
        "  srcs = [],",
        "  outs = ['out/foo1.txt'],",
        "  cmd = 'echo foo1 > $@',",
        ")",
        "genrule(",
        "  name = 'foo2',",
        "  srcs = [],",
        "  outs = ['out/foo2.txt'],",
        "  cmd = 'echo foo2 > $@',",
        ")",
        "genrule(",
        "  name = 'foo3',",
        "  srcs = [],",
        "  outs = ['out/foo3.txt'],",
        "  cmd = 'echo foo3 > $@',",
        ")");
    setDownloadToplevel();

    buildTarget("//:foo1", "//:foo2", "//:foo3");

    assertValidOutputFile("out/foo1.txt", "foo1\n");
    // TODO(chiwang): Make metadata for downloaded outputs local.
    // assertThat(getMetadata("//:foo1").values().stream().noneMatch(FileArtifactValue::isRemote))
    //     .isTrue();
    assertValidOutputFile("out/foo2.txt", "foo2\n");
    // TODO(chiwang): Make metadata for downloaded outputs local.
    // assertThat(getMetadata("//:foo2").values().stream().noneMatch(FileArtifactValue::isRemote))
    //     .isTrue();
    assertValidOutputFile("out/foo3.txt", "foo3\n");
    // TODO(chiwang): Make metadata for downloaded outputs local.
    // assertThat(getMetadata("//:foo3").values().stream().noneMatch(FileArtifactValue::isRemote))
    //     .isTrue();
  }

  @Test
  public void downloadToplevel_incrementalBuild_multipleToplevelTargets() throws Exception {
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo1',",
        "  srcs = [],",
        "  outs = ['out/foo1.txt'],",
        "  cmd = 'echo foo1 > $@',",
        ")",
        "genrule(",
        "  name = 'foo2',",
        "  srcs = [],",
        "  outs = ['out/foo2.txt'],",
        "  cmd = 'echo foo2 > $@',",
        ")",
        "genrule(",
        "  name = 'foo3',",
        "  srcs = [],",
        "  outs = ['out/foo3.txt'],",
        "  cmd = 'echo foo3 > $@',",
        ")");

    buildTarget("//:foo1", "//:foo2", "//:foo3");

    assertOutputsDoNotExist("//:foo1");
    assertThat(getMetadata("//:foo1").values().stream().allMatch(FileArtifactValue::isRemote))
        .isTrue();
    assertOutputsDoNotExist("//:foo2");
    assertThat(getMetadata("//:foo2").values().stream().allMatch(FileArtifactValue::isRemote))
        .isTrue();
    assertOutputsDoNotExist("//:foo3");
    assertThat(getMetadata("//:foo3").values().stream().allMatch(FileArtifactValue::isRemote))
        .isTrue();

    setDownloadToplevel();
    buildTarget("//:foo1", "//:foo2", "//:foo3");

    assertValidOutputFile("out/foo1.txt", "foo1\n");
    // TODO(chiwang): Make metadata for downloaded outputs local.
    // assertThat(getMetadata("//:foo1").values().stream().noneMatch(FileArtifactValue::isRemote))
    //     .isTrue();
    assertValidOutputFile("out/foo2.txt", "foo2\n");
    // TODO(chiwang): Make metadata for downloaded outputs local.
    // assertThat(getMetadata("//:foo2").values().stream().noneMatch(FileArtifactValue::isRemote))
    //     .isTrue();
    assertValidOutputFile("out/foo3.txt", "foo3\n");
    // TODO(chiwang): Make metadata for downloaded outputs local.
    // assertThat(getMetadata("//:foo3").values().stream().noneMatch(FileArtifactValue::isRemote))
    //     .isTrue();
  }

  @Test
  public void downloadToplevel_symlinkFile() throws Exception {
    // TODO(chiwang): Make metadata for downloaded symlink non-remote.
    assumeFalse(OS.getCurrent() == OS.WINDOWS);

    setDownloadToplevel();
    writeSymlinkRule();
    write(
        "BUILD",
        "load(':symlink.bzl', 'symlink')",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo foo > $@',",
        ")",
        "symlink(",
        "  name = 'foo-link',",
        "  target = ':foo'",
        ")");

    buildTarget("//:foo-link");

    assertValidOutputFile("foo-link", "foo\n");

    // Delete link, re-plant symlink
    getOutputPath("foo-link").delete();

    buildTarget("//:foo-link");

    assertValidOutputFile("foo-link", "foo\n");

    // Delete target, re-download it
    getOutputPath("foo").delete();

    assertValidOutputFile("foo-link", "foo\n");
  }

  @Test
  public void downloadToplevel_symlinkSourceFile() throws Exception {
    // TODO(chiwang): Make metadata for downloaded symlink non-remote.
    assumeFalse(OS.getCurrent() == OS.WINDOWS);

    setDownloadToplevel();
    writeSymlinkRule();
    write(
        "BUILD",
        "load(':symlink.bzl', 'symlink')",
        "symlink(",
        "  name = 'foo-link',",
        "  target = ':foo.txt'",
        ")");
    write("foo.txt", "foo");

    buildTarget("//:foo-link");

    assertOnlyOutputContent("//:foo-link", "foo-link", "foo" + lineSeparator());

    // Delete link, re-plant symlink
    getOutputPath("foo-link").delete();

    buildTarget("//:foo-link");

    assertOnlyOutputContent("//:foo-link", "foo-link", "foo" + lineSeparator());
  }

  @Test
  public void downloadToplevel_symlinkTree() throws Exception {
    // TODO(chiwang): Make metadata for downloaded symlink non-remote.
    assumeFalse(OS.getCurrent() == OS.WINDOWS);

    setDownloadToplevel();
    writeSymlinkRule();
    writeOutputDirRule();
    write(
        "BUILD",
        "load(':output_dir.bzl', 'output_dir')",
        "load(':symlink.bzl', 'symlink')",
        "output_dir(",
        "  name = 'foo',",
        "  content_map = {'file-1': '1', 'file-2': '2', 'file-3': '3'},",
        ")",
        "symlink(",
        "  name = 'foo-link',",
        "  target = ':foo'",
        ")");

    buildTarget("//:foo-link");

    assertValidOutputFile("foo-link/file-1", "1");
    assertValidOutputFile("foo-link/file-2", "2");
    assertValidOutputFile("foo-link/file-3", "3");

    getOutputPath("foo-link").deleteTree();

    // Delete link, re-plant symlink
    buildTarget("//:foo-link");

    assertValidOutputFile("foo-link/file-1", "1");
    assertValidOutputFile("foo-link/file-2", "2");
    assertValidOutputFile("foo-link/file-3", "3");

    // Delete target, re-download them
    getOutputPath("foo").deleteTree();

    buildTarget("//:foo-link");

    assertValidOutputFile("foo-link/file-1", "1");
    assertValidOutputFile("foo-link/file-2", "2");
    assertValidOutputFile("foo-link/file-3", "3");
  }

  @Test
  public void treeOutputsFromLocalFileSystem_works() throws Exception {
    // Disable on Windows since it fails for unknown reasons.
    // TODO(chiwang): Enable it on windows.
    if (OS.getCurrent() == OS.WINDOWS) {
      return;
    }
    // Test that tree artifact generated locally can be consumed by other actions.
    // See https://github.com/bazelbuild/bazel/issues/16789

    // Disable remote execution so tree outputs are generated locally
    addOptions("--modify_execution_info=OutputDir=+no-remote-exec");
    setDownloadToplevel();
    writeOutputDirRule();
    write(
        "BUILD",
        "load(':output_dir.bzl', 'output_dir')",
        "output_dir(",
        "  name = 'foo',",
        "  content_map = {'file-1': '1'},",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo)/file-1 > $@ && echo bar >> $@',",
        ")");

    buildTarget("//:foobar");
    waitDownloads();

    assertValidOutputFile("out/foobar.txt", "1bar\n");
  }

  @Test
  public void emptyTreeConsumedByLocalAction() throws Exception {
    // Disable remote execution so that the empty tree artifact is prefetched.
    addOptions("--modify_execution_info=Genrule=+no-remote-exec");
    addOptions("--verbose_failures");
    setDownloadToplevel();
    writeOutputDirRule();
    write(
        "BUILD",
        "load(':output_dir.bzl', 'output_dir')",
        "output_dir(",
        "  name = 'foo',",
        "  content_map = {},", // no files
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['foobar.txt'],",
        "  cmd = 'touch $@',",
        ")");

    buildTarget("//:foobar");
    waitDownloads();
  }

  @Test
  public void multiplePackagePaths_buildsSuccessfully() throws Exception {
    write(
        "../a/src/BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo foo > $@',",
        ")");
    write(
        "BUILD",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = ['//src:foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location //src:foo) > $@ && echo bar >> $@',",
        ")");
    addOptions("--package_path=%workspace%:%workspace%/../a");
    setDownloadToplevel();

    buildTarget("//:foobar");
    waitDownloads();

    assertValidOutputFile("out/foobar.txt", "foo\nbar\n");
  }

  @Test
  public void incrementalBuild_deleteOutputsInUnwritableParentDirectory() throws Exception {
    write(
        "BUILD",
        "genrule(",
        "  name = 'unwritable',",
        "  srcs = ['file.in'],",
        "  outs = ['unwritable/somefile.out'],",
        "  cmd = 'cat $(SRCS) > $@; chmod a-w $$(dirname $@)',",
        "  local = True,",
        ")");
    write("file.in", "content");
    buildTarget("//:unwritable");

    write("file.in", "updated content");

    buildTarget("//:unwritable");
  }

  @Test
  public void incrementalBuild_treeArtifacts_correctlyProducesNewTree() throws Exception {
    // Disable on Windows since it fails for unknown reasons.
    // TODO(chiwang): Enable it on windows.
    if (OS.getCurrent() == OS.WINDOWS) {
      return;
    }
    writeOutputDirRule();
    write(
        "BUILD",
        "load(':output_dir.bzl', 'output_dir')",
        "output_dir(",
        "  name = 'foo',",
        "  content_map = {'file-1': '1', 'file-2': '2', 'file-3': '3'},",
        ")");
    setDownloadToplevel();
    buildTarget("//:foo");
    waitDownloads();

    write(
        "BUILD",
        "load(':output_dir.bzl', 'output_dir')",
        "output_dir(",
        "  name = 'foo',",
        "  content_map = {'file-1': '1', 'file-4': '4'},",
        ")");
    restartServer();
    setDownloadToplevel();
    buildTarget("//:foo");
    waitDownloads();

    assertValidOutputFile("foo/file-1", "1");
    assertValidOutputFile("foo/file-4", "4");
    assertOutputDoesNotExist("foo/file-2");
    assertOutputDoesNotExist("foo/file-3");
  }

  @Test
  public void incrementalBuild_restartServer_hitActionCache() throws Exception {
    // Prepare workspace
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo foo > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        ")");
    ActionEventCollector actionEventCollector = new ActionEventCollector();
    getRuntimeWrapper().registerSubscriber(actionEventCollector);

    // Clean build
    buildTarget("//:foobar");

    // all action should be executed
    assertThat(actionEventCollector.getActionExecutedEvents()).hasSize(3);
    // no outputs are staged
    assertOutputsDoNotExist("//:foobar");

    restartServer();
    actionEventCollector = new ActionEventCollector();
    getRuntimeWrapper().registerSubscriber(actionEventCollector);

    // Incremental build
    buildTarget("//:foobar");

    // all actions should hit the action cache.
    assertThat(actionEventCollector.getActionExecutedEvents()).isEmpty();
    // no outputs are staged
    assertOutputsDoNotExist("//:foobar");
  }

  @Test
  public void incrementalBuild_sourceModified_rerunActions() throws Exception {
    // Arrange: Prepare workspace and run a clean build
    write("foo.in", "foo");
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = ['foo.in'],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'cat $(SRCS) > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        "  tags = ['no-remote'],",
        ")");

    buildTarget("//:foobar");
    assertValidOutputFile("out/foo.txt", "foo" + lineSeparator());
    assertValidOutputFile("out/foobar.txt", "foo" + lineSeparator() + "bar\n");

    // Act: Modify source file and run an incremental build
    write("foo.in", "modified");

    ActionEventCollector actionEventCollector = new ActionEventCollector();
    getRuntimeWrapper().registerSubscriber(actionEventCollector);
    buildTarget("//:foobar");

    // Assert: All actions transitively depend on the source file are re-executed and outputs are
    // correct.
    assertValidOutputFile("out/foo.txt", "modified" + lineSeparator());
    assertValidOutputFile("out/foobar.txt", "modified" + lineSeparator() + "bar\n");
    assertThat(actionEventCollector.getNumActionNodesEvaluated()).isEqualTo(2);
  }

  @Test
  public void incrementalBuild_intermediateOutputDeleted_nothingIsReEvaluated() throws Exception {
    setIncompatibleWithSkymeld();
    // Arrange: Prepare workspace and run a clean build
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo foo > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        "  tags = ['no-remote'],",
        ")");

    buildTarget("//:foobar");
    assertValidOutputFile("out/foo.txt", "foo\n");
    assertValidOutputFile("out/foobar.txt", "foo\nbar\n");

    // Act: Delete intermediate output and run an incremental build
    var fooPath = getOutputPath("out/foo.txt");
    fooPath.delete();

    ActionEventCollector actionEventCollector = new ActionEventCollector();
    getRuntimeWrapper().registerSubscriber(actionEventCollector);
    buildTarget("//:foobar");

    // Assert: local output is deleted, skyframe should trust remote files so no nodes will be
    // re-evaluated.
    assertOutputDoesNotExist("out/foo.txt");
    assertValidOutputFile("out/foobar.txt", "foo\nbar\n");
    assertThat(actionEventCollector.getNumActionNodesEvaluated()).isEqualTo(0);
  }

  @Test
  public void incrementalBuild_remoteFileMetadataIsReplacedWithLocalFileMetadata()
      throws Exception {
    // We need to download the intermediate output
    if (!hasAccessToRemoteOutputs()) {
      return;
    }

    // Arrange: Prepare workspace and run a clean build
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo foo > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        "  tags = ['no-remote'],",
        ")");

    buildTarget("//:foobar");
    assertValidOutputFile("out/foo.txt", "foo\n");
    assertValidOutputFile("out/foobar.txt", "foo\nbar\n");
    assertThat(getOnlyElement(getMetadata("//:foo").values()).isRemote()).isTrue();

    // Act: Do an incremental build without any modifications
    ActionEventCollector actionEventCollector = new ActionEventCollector();
    getRuntimeWrapper().registerSubscriber(actionEventCollector);
    buildTarget("//:foobar");

    // Assert: remote file metadata is replaced with local file metadata
    assertValidOutputFile("out/foo.txt", "foo\n");
    assertValidOutputFile("out/foobar.txt", "foo\nbar\n");
    assertThat(actionEventCollector.getActionExecutedEvents()).isEmpty();
    // Two actions are invalidated but were able to hit the action cache
    assertThat(actionEventCollector.getCachedActionEvents()).hasSize(2);
    assertThat(getOnlyElement(getMetadata("//:foo").values()).isRemote()).isFalse();
  }

  protected ImmutableMap<Artifact, FileArtifactValue> getMetadata(String target) throws Exception {
    var result = ImmutableMap.<Artifact, FileArtifactValue>builder();
    var evaluator = getRuntimeWrapper().getSkyframeExecutor().getEvaluator();
    for (var artifact : getArtifacts(target)) {
      var value = evaluator.getExistingValue(Artifact.key(artifact));
      if (value instanceof ActionExecutionValue) {
        result.putAll(((ActionExecutionValue) value).getAllFileValues());
      } else if (value instanceof TreeArtifactValue) {
        result.putAll(((TreeArtifactValue) value).getChildValues());
      }
    }
    return result.buildOrThrow();
  }

  protected FileArtifactValue getMetadata(Artifact output) throws Exception {
    var evaluator = getRuntimeWrapper().getSkyframeExecutor().getEvaluator();
    var value = evaluator.getExistingValue(Artifact.key(output));
    if (value instanceof ActionExecutionValue) {
      return ((ActionExecutionValue) value).getAllFileValues().get(output);
    } else if (value instanceof TreeArtifactValue) {
      return ((TreeArtifactValue) value).getChildValues().get(output);
    }
    return null;
  }

  @Test
  public void incrementalBuild_intermediateOutputModified_rerunGeneratingActions()
      throws Exception {
    // Arrange: Prepare workspace and run a clean build
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['out/foo.txt'],",
        "  cmd = 'echo foo > $@',",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo'],",
        "  outs = ['out/foobar.txt'],",
        "  cmd = 'cat $(location :foo) > $@ && echo bar >> $@',",
        "  tags = ['no-remote'],",
        ")");

    buildTarget("//:foobar");
    assertValidOutputFile("out/foo.txt", "foo\n");
    assertValidOutputFile("out/foobar.txt", "foo\nbar\n");

    // Act: Modify the intermediate output and run a incremental build
    var fooPath = getOutputPath("out/foo.txt");
    fooPath.delete();
    writeAbsolute(fooPath, "modified");

    ActionEventCollector actionEventCollector = new ActionEventCollector();
    getRuntimeWrapper().registerSubscriber(actionEventCollector);
    buildTarget("//:foobar");

    // Assert: the stale intermediate file should be deleted by skyframe before executing the
    // generating action. Since download minimal, the output didn't get downloaded. Since the input
    // to action :foobar didn't change, we hit the skyframe cache, so the action node didn't event
    // get evaluated. The input didn't get prefetched neither.
    assertOutputDoesNotExist("out/foo.txt");
    assertValidOutputFile("out/foobar.txt", "foo\nbar\n");
    assertThat(actionEventCollector.getActionExecutedEvents()).hasSize(1);
    assertThat(actionEventCollector.getCachedActionEvents()).isEmpty();
    var executedAction = actionEventCollector.getActionExecutedEvents().get(0).getAction();
    assertThat(executedAction.getPrimaryOutput().getFilename()).isEqualTo("foo.txt");
  }

  @Test
  public void remoteCacheEvictBlobs_whenPrefetchingInputFile_incrementalBuildCanContinue()
      throws Exception {
    setIncompatibleWithSkymeld();
    // Arrange: Prepare workspace and populate remote cache
    write(
        "a/BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = ['foo.in'],",
        "  outs = ['foo.out'],",
        "  cmd = 'cat $(SRCS) > $@',",
        ")",
        "genrule(",
        "  name = 'bar',",
        "  srcs = ['foo.out', 'bar.in'],",
        "  outs = ['bar.out'],",
        "  cmd = 'cat $(SRCS) > $@',",
        ")");
    write("a/foo.in", "foo");
    write("a/bar.in", "bar");

    // Populate remote cache
    buildTarget("//a:bar");
    getOutputPath("a/foo.out").delete();
    getOutputPath("a/bar.out").delete();
    getOutputBase().getRelative("action_cache").deleteTreesBelow();
    restartServer();
    setIncompatibleWithSkymeld();

    // Clean build, foo.out isn't downloaded
    buildTarget("//a:bar");
    assertOutputDoesNotExist("a/foo.out");

    // Evict blobs from remote cache
    evictAllBlobs();

    // trigger build error
    write("a/bar.in", "updated bar");
    addOptions("--strategy_regexp=.*bar=local");
    // Build failed because of remote cache eviction
    assertThrows(BuildFailedException.class, () -> buildTarget("//a:bar"));

    // Act: Do an incremental build without "clean" or "shutdown"
    buildTarget("//a:bar");

    // Assert: target was successfully built
    assertValidOutputFile("a/bar.out", "foo" + lineSeparator() + "updated bar" + lineSeparator());
  }

  @Test
  public void remoteCacheEvictBlobs_whenPrefetchingInputTree_incrementalBuildCanContinue()
      throws Exception {
    setIncompatibleWithSkymeld();
    // Arrange: Prepare workspace and populate remote cache
    write("BUILD");
    writeOutputDirRule();
    write(
        "a/BUILD",
        "load('//:output_dir.bzl', 'output_dir')",
        "output_dir(",
        "  name = 'foo.out',",
        "  content_map = {'file-inside': 'hello world'},",
        ")",
        "genrule(",
        "  name = 'bar',",
        "  srcs = ['foo.out', 'bar.in'],",
        "  outs = ['bar.out'],",
        "  cmd = '( ls $(location :foo.out); cat $(location :bar.in) ) > $@',",
        ")");
    write("a/bar.in", "bar");

    // Populate remote cache
    buildTarget("//a:bar");
    getOutputPath("a/foo.out").deleteTreesBelow();
    getOutputPath("a/bar.out").delete();
    getOutputBase().getRelative("action_cache").deleteTreesBelow();
    restartServer();
    setIncompatibleWithSkymeld();

    // Clean build, foo.out isn't downloaded
    buildTarget("//a:bar");
    assertOutputDoesNotExist("a/foo.out/file-inside");

    // Evict blobs from remote cache
    evictAllBlobs();

    // trigger build error
    write("a/bar.in", "updated bar");
    addOptions("--strategy_regexp=.*bar=local");
    // Build failed because of remote cache eviction
    assertThrows(BuildFailedException.class, () -> buildTarget("//a:bar"));

    // Act: Do an incremental build without "clean" or "shutdown"
    buildTarget("//a:bar");

    // Assert: target was successfully built
    assertValidOutputFile("a/bar.out", "file-inside\nupdated bar" + lineSeparator());
  }

  @Test
  public void remoteFilesExpiredBetweenBuilds_rerunGeneratingActions() throws Exception {
    // Arrange: Prepare workspace and populate remote cache
    write(
        "a/BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = ['foo.in'],",
        "  outs = ['foo.out'],",
        "  cmd = 'cat $(SRCS) > $@',",
        ")",
        "genrule(",
        "  name = 'bar',",
        "  srcs = ['foo.out', 'bar.in'],",
        "  outs = ['bar.out'],",
        "  cmd = 'cat $(SRCS) > $@',",
        ")");
    write("a/foo.in", "foo");
    write("a/bar.in", "bar");

    // Populate remote cache
    buildTarget("//a:bar");
    getOutputPath("a/foo.out").delete();
    getOutputPath("a/bar.out").delete();
    getOutputBase().getRelative("action_cache").deleteTreesBelow();
    restartServer();

    // Clean build, foo.out isn't downloaded
    setDownloadToplevel();
    addOptions("--experimental_remote_cache_ttl=0s");
    buildTarget("//a:bar");
    assertOutputDoesNotExist("a/foo.out");

    // Evict blobs from remote cache
    evictAllBlobs();

    // Act: Do an incremental build
    write("a/bar.in", "updated bar");
    addOptions("--strategy_regexp=.*bar=local");
    buildTarget("//a:bar");
    waitDownloads();

    // Assert: target was successfully built
    assertValidOutputFile("a/bar.out", "foo" + lineSeparator() + "updated bar" + lineSeparator());
  }

  @Test
  public void remoteTreeFilesExpiredBetweenBuilds_rerunGeneratingActions() throws Exception {
    // Arrange: Prepare workspace and populate remote cache
    write("BUILD");
    writeOutputDirRule();
    write(
        "a/BUILD",
        "load('//:output_dir.bzl', 'output_dir')",
        "output_dir(",
        "  name = 'foo.out',",
        "  content_map = {'file-inside': 'hello world'},",
        ")",
        "genrule(",
        "  name = 'bar',",
        "  srcs = ['foo.out', 'bar.in'],",
        "  outs = ['bar.out'],",
        "  cmd = '( ls $(location :foo.out); cat $(location :bar.in) ) > $@',",
        ")");
    write("a/bar.in", "bar");

    // Populate remote cache
    buildTarget("//a:bar");
    getOutputPath("a/foo.out").deleteTreesBelow();
    getOutputPath("a/bar.out").delete();
    getOutputBase().getRelative("action_cache").deleteTreesBelow();
    restartServer();

    // Clean build, foo.out isn't downloaded
    setDownloadToplevel();
    addOptions("--experimental_remote_cache_ttl=0s");
    buildTarget("//a:bar");
    assertOutputDoesNotExist("a/foo.out/file-inside");

    // Evict blobs from remote cache
    evictAllBlobs();

    // Act: Do an incremental build
    write("a/bar.in", "updated bar");
    addOptions("--strategy_regexp=.*bar=local");
    buildTarget("//a:bar");
    waitDownloads();

    // Assert: target was successfully built
    assertValidOutputFile("a/bar.out", "file-inside\nupdated bar" + lineSeparator());
  }

  @Test
  public void nonDeclaredSymlinksFromLocalActions() throws Exception {
    write(
        "BUILD",
        "genrule(",
        "  name = 'foo',",
        "  srcs = [],",
        "  outs = ['foo.txt'],",
        "  cmd = 'echo foo > $@',",
        ")",
        "genrule(",
        "  name = 'foo-link',",
        "  srcs = [':foo'],",
        "  outs = ['foo.link'],",
        "  cmd = 'ln -s foo.txt $@',",
        "  local = True,",
        ")",
        "genrule(",
        "  name = 'foobar',",
        "  srcs = [':foo-link'],",
        "  outs = ['foobar.txt'],",
        "  cmd = 'cat $(location :foo-link) > $@ && echo bar >> $@',",
        "  local = True,",
        ")");

    buildTarget("//:foobar");

    assertValidOutputFile("foobar.txt", "foo\nbar\n");
  }

  protected void assertOutputsDoNotExist(String target) throws Exception {
    for (Artifact output : getArtifacts(target)) {
      assertWithMessage(
              "output %s for target %s should not exist", output.getExecPathString(), target)
          .that(output.getPath().exists())
          .isFalse();
    }
  }

  protected Path getOutputPath(String binRelativePath) {
    return getDirectories()
        .getWorkspace()
        .getRelative(getDirectories().getProductName() + "-bin")
        .getRelative(binRelativePath);
  }

  protected void assertOutputDoesNotExist(String binRelativePath) {
    Path output = getOutputPath(binRelativePath);
    assertThat(output.exists()).isFalse();
  }

  protected void assertOnlyOutputContent(String target, String filename, String content)
      throws Exception {
    Artifact output = getOnlyElement(getArtifacts(target));
    assertThat(output.getFilename()).isEqualTo(filename);
    assertThat(output.getPath().exists()).isTrue();
    assertOutputEquals(output.getPath(), content);
  }

  protected void assertValidOutputFile(String binRelativePath, String content) throws Exception {
    Path output = getOutputPath(binRelativePath);
    assertOutputEquals(getOutputPath(binRelativePath), content);
    assertThat(output.isReadable()).isTrue();
    assertThat(output.isWritable()).isFalse();
    assertThat(output.isExecutable()).isTrue();
  }

  protected void writeSymlinkRule() throws IOException {
    write(
        "symlink.bzl",
        "def _symlink_impl(ctx):",
        "  target = ctx.file.target",
        "  if target.is_directory:",
        "    link = ctx.actions.declare_directory(ctx.attr.name)",
        "  else:",
        "    link = ctx.actions.declare_file(ctx.attr.name)",
        "  ctx.actions.symlink(output = link, target_file = target)",
        "  return DefaultInfo(files = depset([link]))",
        "",
        "symlink = rule(",
        "  implementation = _symlink_impl,",
        "  attrs = {",
        "    'target': attr.label(mandatory = True, allow_single_file = True),",
        "  }",
        ")");
  }

  protected void writeOutputDirRule() throws IOException {
    write(
        "output_dir.bzl",
        "def _output_dir_impl(ctx):",
        "  out = ctx.actions.declare_directory(ctx.attr.name)",
        "  args = []",
        "  for name, content in ctx.attr.content_map.items():",
        "    args.append(out.path + '/' + name)",
        "    args.append(content)",
        "  ctx.actions.run_shell(",
        "    mnemonic = 'OutputDir',",
        "    outputs = [out],",
        "    arguments = args,",
        "    command = 'while (($#)); do echo -n \"$2\" > $1; shift 2; done',",
        "  )",
        "  return DefaultInfo(files = depset([out]))",
        "",
        "output_dir = rule(",
        "  implementation = _output_dir_impl,",
        "  attrs = {",
        "    'content_map': attr.string_dict(mandatory = True),",
        "  },",
        ")");
  }

  protected void writeCopyAspectRule(boolean aggregate) throws IOException {
    var lines = ImmutableList.<String>builder();
    lines.add(
        "def _copy_aspect_impl(target, ctx):",
        "  files = []",
        "  for src in ctx.rule.files.srcs:",
        "    dst = ctx.actions.declare_file(src.basename + '.copy')",
        "    ctx.actions.run_shell(",
        "      inputs = [src],",
        "      outputs = [dst],",
        "      command = '''",
        "cp $1 $2",
        "''',",
        "      arguments = [src.path, dst.path],",
        "    )",
        "    files.append(dst)",
        "");
    if (aggregate) {
      lines.add(
          "  files = depset(",
          "    direct = files,",
          "    transitive = [src[OutputGroupInfo].copy for src in ctx.rule.attr.srcs if"
              + " OutputGroupInfo in src],",
          "  )");
    } else {
      lines.add("  files = depset(files)");
    }
    lines.add(
        "",
        "  return [OutputGroupInfo(copy = files)]",
        "",
        "copy_aspect = aspect(",
        "  implementation = _copy_aspect_impl,",
        "  attr_aspects = ['srcs'],",
        ")");
    write("rules.bzl", lines.build().toArray(new String[0]));
  }

  protected static class ActionEventCollector {
    private final List<ActionExecutedEvent> actionExecutedEvents = new ArrayList<>();
    private final List<CachedActionEvent> cachedActionEvents = new ArrayList<>();

    @Subscribe
    public void onActionExecuted(ActionExecutedEvent event) {
      actionExecutedEvents.add(event);
    }

    @Subscribe
    public void onCachedAction(CachedActionEvent event) {
      cachedActionEvents.add(event);
    }

    public int getNumActionNodesEvaluated() {
      return getActionExecutedEvents().size() + getCachedActionEvents().size();
    }

    public void clear() {
      this.actionExecutedEvents.clear();
      this.cachedActionEvents.clear();
    }

    public List<ActionExecutedEvent> getActionExecutedEvents() {
      return actionExecutedEvents;
    }

    public List<CachedActionEvent> getCachedActionEvents() {
      return cachedActionEvents;
    }
  }

  protected void restartServer() throws Exception {
    // Simulates a server restart
    createRuntimeWrapper();
  }

  protected static String lineSeparator() {
    return System.getProperty("line.separator");
  }
}