src/test/java/com/google/devtools/build/lib/remote/RemoteCircuitBreakerRecoveryIntegrationTest.java - bazel - Git at Google

 // Copyright 2026 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 package com.google.devtools.build.lib.remote;

 import static com.google.common.truth.Truth.assertThat;
 import static com.google.devtools.build.lib.vfs.FileSystemUtils.readContent;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertThrows;

 import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.actions.BuildFailedException;
 import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
 import com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase;
 import com.google.devtools.build.lib.remote.options.RemoteStartupOptions;
 import com.google.devtools.build.lib.remote.util.IntegrationTestUtils;
 import com.google.devtools.build.lib.remote.util.IntegrationTestUtils.WorkerInstance;
 import com.google.devtools.build.lib.runtime.BlazeModule;
 import com.google.devtools.build.lib.runtime.BlazeRuntime;
 import com.google.devtools.build.lib.runtime.BlockWaitingModule;
 import com.google.devtools.build.lib.runtime.BuildSummaryStatsModule;
 import com.google.devtools.build.lib.standalone.StandaloneModule;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.common.options.OptionsBase;
 import org.junit.After;
 import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;

 /**
  * Integration tests for the remote failure circuit breaker's opt-in half-open recovery, run under
  * {@code --remote_download_minimal} so that tripping the breaker happens while prefetching a
  * remote-only input.
  *
  * <p>Kept separate from {@link BuildWithoutTheBytesIntegrationTest} because these tests need a
  * dedicated remote worker that injects transient read failures, whereas every other
  * Build-without-the-Bytes test runs against a pristine worker.
  */
 @RunWith(JUnit4.class)
 public class RemoteCircuitBreakerRecoveryIntegrationTest extends BuildIntegrationTestCase {

   // A worker that returns UNAVAILABLE for the first armed ByteStream.Read, so a build can trip the
   // remote failure circuit breaker on a transient, self-healing failure.
   @ClassRule @Rule
   public static final WorkerInstance worker =
       IntegrationTestUtils.createFailFirstReadWorker(/* n= */ 1);

   // The worker logs this to stderr each time it injects a read failure (see FailFirstNInterceptor
   // in RemoteWorker). The tests match on it to prove a transient failure was actually delivered;
   // keep it in sync with the worker's message.
   private static final String INJECTED_READ_FAILURE_LOG =
       "INJECTED_UNAVAILABLE google.bytestream.ByteStream/Read";

   @Override
   protected ImmutableList<Class<? extends OptionsBase>> getStartupOptionClasses() {
     return ImmutableList.<Class<? extends OptionsBase>>builder()
         .addAll(super.getStartupOptionClasses())
         .add(RemoteStartupOptions.class)
         .build();
   }

   @Override
   protected void setupOptions() throws Exception {
     super.setupOptions();
     addOptions(
         "--remote_executor=grpc://localhost:" + worker.getPort(),
         "--remote_download_minimal",
         // Both tests recover a lost input via action rewinding: a failed prefetch (with
         // --remote_retries=0) is treated as a lost input, and rewinding re-runs the action.
         "--rewind_lost_inputs",
         // Disable build-level invocation retries (default 5) so recovery is exercised via action
         // rewinding within one build, not by retrying the whole build with a fresh breaker.
         "--experimental_remote_cache_eviction_retries=0");
   }

   @Override
   protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
     return super.getRuntimeBuilder()
         .addBlazeModule(new RemoteModule())
         .addBlazeModule(new BuildSummaryStatsModule())
         .addBlazeModule(new BlockWaitingModule());
   }

   @Override
   protected ImmutableList<BlazeModule> getSpawnModules() {
     return ImmutableList.<BlazeModule>builder()
         .addAll(super.getSpawnModules())
         .add(new StandaloneModule())
         .add(new CredentialModule())
         .build();
   }

   @After
   public void waitDownloads() throws Exception {
     runtimeWrapper.newCommand();
   }

   private Path getOutputPath(String binRelativePath) {
     return getTargetConfiguration().getBinDir().getRoot().getRelative(binRelativePath);
   }

   private void assertValidOutputFile(String binRelativePath, String content) throws Exception {
     Path output = getOutputPath(binRelativePath);
     assertThat(readContent(output, UTF_8)).isEqualTo(content);
     assertThat(output.isReadable()).isTrue();
     assertThat(output.isWritable()).isFalse();
     assertThat(output.isExecutable()).isTrue();
   }

   private void writeFooBar() throws Exception {
     write(
         "a/BUILD",
         """
         genrule(
             name = "foo",
             srcs = ["foo.in"],
             outs = ["foo.out"],
             cmd = "cat $(SRCS) > $@",
         )

         genrule(
             name = "bar",
             srcs = [
                 "foo.out",
                 "bar.in",
             ],
             outs = ["bar.out"],
             cmd = "cat $(SRCS) > $@",
             tags = ["no-remote-exec"],
         )
         """);
     write("a/foo.in", "foo");
     write("a/bar.in", "bar");
   }

   /**
    * Populates the remote cache and leaves foo.out remote-only, so a later build of //a:bar must
    * prefetch it. The failing worker is wired up in {@link #setupOptions}, which {@link
    * #createRuntimeWrapper} re-runs, so foo.out stays remote-only across the simulated server
    * restart.
    */
   private void setupRemoteOnlyFooOut() throws Exception {
     writeFooBar();

     buildTarget("//a:bar");
     getOutputPath("a/foo.out").delete();
     getOutputPath("a/bar.out").delete();
     getOutputBase().getRelative("action_cache").deleteTreesBelow();
     createRuntimeWrapper(); // Simulate a server restart.

     buildTarget("//a:bar");
     assertThat(getOutputPath("a/foo.out").exists()).isFalse();
   }

   /**
    * Asserts the worker delivered at least one injected read failure since {@code
    * previousStderrLength} (captured just before the build) — i.e. the breaker actually saw a
    * transient remote failure, so a green build is meaningful rather than vacuous.
    */
   private static void assertReadFailureInjectedSince(int previousStderrLength) {
     assertThat(worker.getStderr().substring(previousStderrLength))
         .contains(INJECTED_READ_FAILURE_LOG);
   }

   @Test
   public void recovers_whenPrefetchingInput_succeedsWithActionRewinding() throws Exception {
     setupRemoteOnlyFooOut();

     // Act: enable the failure breaker with a short recovery delay, arm a single transient read
     // failure, and force bar to re-prefetch the remote-only foo.out. The prefetch read trips the
     // breaker; with --remote_retries=0 that becomes a lost input and drives action rewinding, and
     // the breaker re-closes on a trial probe during the rewind so the build recovers.
     addOptions(
         "--experimental_circuit_breaker_strategy=failure",
         "--experimental_remote_min_fail_count_to_compute_failure_rate=1",
         "--experimental_remote_min_call_count_to_compute_failure_rate=1",
         "--experimental_remote_failure_rate_threshold=1",
         "--experimental_remote_circuit_breaker_recovery_delay=1ms",
         "--remote_retries=0");
     write("a/bar.in", "updated bar");
     int stderrLenBefore = worker.getStderr().length();
     worker.armReadFailures();

     buildTarget("//a:bar");

     // Assert: a transient failure was actually delivered (the breaker tripped) and, thanks to
     // recovery, the build still succeeded with the correct output.
     assertReadFailureInjectedSince(stderrLenBefore);
     assertValidOutputFile("a/bar.out", "foo\nupdated bar\n");
   }

   @Test
   public void trips_whenPrefetchingInput_recoveryDisabled_fails() throws Exception {
     setupRemoteOnlyFooOut();

     // Act: same as the recovery test, but recovery is disabled (the delay defaults to 0), so the
     // tripped breaker stays open for the rest of the build and the rewound re-execution is
     // rejected.
     addOptions(
         "--experimental_circuit_breaker_strategy=failure",
         "--experimental_remote_min_fail_count_to_compute_failure_rate=1",
         "--experimental_remote_min_call_count_to_compute_failure_rate=1",
         "--experimental_remote_failure_rate_threshold=1",
         "--remote_retries=0");
     write("a/bar.in", "updated bar");
     int stderrLenBefore = worker.getStderr().length();
     worker.armReadFailures();

     var error = assertThrows(BuildFailedException.class, () -> buildTarget("//a:bar"));

     // Assert: the breaker tripped and, without recovery, the build failed with a remote error.
     assertReadFailureInjectedSince(stderrLenBefore);
     assertThat(error.getDetailedExitCode().getExitCode().getNumericExitCode()).isEqualTo(34);
   }
 }
	// Copyright 2026 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.
	package com.google.devtools.build.lib.remote;

	import static com.google.common.truth.Truth.assertThat;
	import static com.google.devtools.build.lib.vfs.FileSystemUtils.readContent;
	import static java.nio.charset.StandardCharsets.UTF_8;
	import static org.junit.Assert.assertThrows;

	import com.google.common.collect.ImmutableList;
	import com.google.devtools.build.lib.actions.BuildFailedException;
	import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
	import com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase;
	import com.google.devtools.build.lib.remote.options.RemoteStartupOptions;
	import com.google.devtools.build.lib.remote.util.IntegrationTestUtils;
	import com.google.devtools.build.lib.remote.util.IntegrationTestUtils.WorkerInstance;
	import com.google.devtools.build.lib.runtime.BlazeModule;
	import com.google.devtools.build.lib.runtime.BlazeRuntime;
	import com.google.devtools.build.lib.runtime.BlockWaitingModule;
	import com.google.devtools.build.lib.runtime.BuildSummaryStatsModule;
	import com.google.devtools.build.lib.standalone.StandaloneModule;
	import com.google.devtools.build.lib.vfs.Path;
	import com.google.devtools.common.options.OptionsBase;
	import org.junit.After;
	import org.junit.ClassRule;
	import org.junit.Rule;
	import org.junit.Test;
	import org.junit.runner.RunWith;
	import org.junit.runners.JUnit4;

	/**
	* Integration tests for the remote failure circuit breaker's opt-in half-open recovery, run under
	* {@code --remote_download_minimal} so that tripping the breaker happens while prefetching a
	* remote-only input.
	*
	* <p>Kept separate from {@link BuildWithoutTheBytesIntegrationTest} because these tests need a
	* dedicated remote worker that injects transient read failures, whereas every other
	* Build-without-the-Bytes test runs against a pristine worker.
	*/
	@RunWith(JUnit4.class)
	public class RemoteCircuitBreakerRecoveryIntegrationTest extends BuildIntegrationTestCase {

	// A worker that returns UNAVAILABLE for the first armed ByteStream.Read, so a build can trip the
	// remote failure circuit breaker on a transient, self-healing failure.
	@ClassRule @Rule
	public static final WorkerInstance worker =
	IntegrationTestUtils.createFailFirstReadWorker(/* n= */ 1);

	// The worker logs this to stderr each time it injects a read failure (see FailFirstNInterceptor
	// in RemoteWorker). The tests match on it to prove a transient failure was actually delivered;
	// keep it in sync with the worker's message.
	private static final String INJECTED_READ_FAILURE_LOG =
	"INJECTED_UNAVAILABLE google.bytestream.ByteStream/Read";

	@Override
	protected ImmutableList<Class<? extends OptionsBase>> getStartupOptionClasses() {
	return ImmutableList.<Class<? extends OptionsBase>>builder()
	.addAll(super.getStartupOptionClasses())
	.add(RemoteStartupOptions.class)
	.build();
	}

	@Override
	protected void setupOptions() throws Exception {
	super.setupOptions();
	addOptions(
	"--remote_executor=grpc://localhost:" + worker.getPort(),
	"--remote_download_minimal",
	// Both tests recover a lost input via action rewinding: a failed prefetch (with
	// --remote_retries=0) is treated as a lost input, and rewinding re-runs the action.
	"--rewind_lost_inputs",
	// Disable build-level invocation retries (default 5) so recovery is exercised via action
	// rewinding within one build, not by retrying the whole build with a fresh breaker.
	"--experimental_remote_cache_eviction_retries=0");
	}

	@Override
	protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
	return super.getRuntimeBuilder()
	.addBlazeModule(new RemoteModule())
	.addBlazeModule(new BuildSummaryStatsModule())
	.addBlazeModule(new BlockWaitingModule());
	}

	@Override
	protected ImmutableList<BlazeModule> getSpawnModules() {
	return ImmutableList.<BlazeModule>builder()
	.addAll(super.getSpawnModules())
	.add(new StandaloneModule())
	.add(new CredentialModule())
	.build();
	}

	@After
	public void waitDownloads() throws Exception {
	runtimeWrapper.newCommand();
	}

	private Path getOutputPath(String binRelativePath) {
	return getTargetConfiguration().getBinDir().getRoot().getRelative(binRelativePath);
	}

	private void assertValidOutputFile(String binRelativePath, String content) throws Exception {
	Path output = getOutputPath(binRelativePath);
	assertThat(readContent(output, UTF_8)).isEqualTo(content);
	assertThat(output.isReadable()).isTrue();
	assertThat(output.isWritable()).isFalse();
	assertThat(output.isExecutable()).isTrue();
	}

	private void writeFooBar() throws Exception {
	write(
	"a/BUILD",
	"""
	genrule(
	name = "foo",
	srcs = ["foo.in"],
	outs = ["foo.out"],
	cmd = "cat $(SRCS) > $@",
	)

	genrule(
	name = "bar",
	srcs = [
	"foo.out",
	"bar.in",
	],
	outs = ["bar.out"],
	cmd = "cat $(SRCS) > $@",
	tags = ["no-remote-exec"],
	)
	""");
	write("a/foo.in", "foo");
	write("a/bar.in", "bar");
	}

	/**
	* Populates the remote cache and leaves foo.out remote-only, so a later build of //a:bar must
	* prefetch it. The failing worker is wired up in {@link #setupOptions}, which {@link
	* #createRuntimeWrapper} re-runs, so foo.out stays remote-only across the simulated server
	* restart.
	*/
	private void setupRemoteOnlyFooOut() throws Exception {
	writeFooBar();

	buildTarget("//a:bar");
	getOutputPath("a/foo.out").delete();
	getOutputPath("a/bar.out").delete();
	getOutputBase().getRelative("action_cache").deleteTreesBelow();
	createRuntimeWrapper(); // Simulate a server restart.

	buildTarget("//a:bar");
	assertThat(getOutputPath("a/foo.out").exists()).isFalse();
	}

	/**
	* Asserts the worker delivered at least one injected read failure since {@code
	* previousStderrLength} (captured just before the build) — i.e. the breaker actually saw a
	* transient remote failure, so a green build is meaningful rather than vacuous.
	*/
	private static void assertReadFailureInjectedSince(int previousStderrLength) {
	assertThat(worker.getStderr().substring(previousStderrLength))
	.contains(INJECTED_READ_FAILURE_LOG);
	}

	@Test
	public void recovers_whenPrefetchingInput_succeedsWithActionRewinding() throws Exception {
	setupRemoteOnlyFooOut();

	// Act: enable the failure breaker with a short recovery delay, arm a single transient read
	// failure, and force bar to re-prefetch the remote-only foo.out. The prefetch read trips the
	// breaker; with --remote_retries=0 that becomes a lost input and drives action rewinding, and
	// the breaker re-closes on a trial probe during the rewind so the build recovers.
	addOptions(
	"--experimental_circuit_breaker_strategy=failure",
	"--experimental_remote_min_fail_count_to_compute_failure_rate=1",
	"--experimental_remote_min_call_count_to_compute_failure_rate=1",
	"--experimental_remote_failure_rate_threshold=1",
	"--experimental_remote_circuit_breaker_recovery_delay=1ms",
	"--remote_retries=0");
	write("a/bar.in", "updated bar");
	int stderrLenBefore = worker.getStderr().length();
	worker.armReadFailures();

	buildTarget("//a:bar");

	// Assert: a transient failure was actually delivered (the breaker tripped) and, thanks to
	// recovery, the build still succeeded with the correct output.
	assertReadFailureInjectedSince(stderrLenBefore);
	assertValidOutputFile("a/bar.out", "foo\nupdated bar\n");
	}

	@Test
	public void trips_whenPrefetchingInput_recoveryDisabled_fails() throws Exception {
	setupRemoteOnlyFooOut();

	// Act: same as the recovery test, but recovery is disabled (the delay defaults to 0), so the
	// tripped breaker stays open for the rest of the build and the rewound re-execution is
	// rejected.
	addOptions(
	"--experimental_circuit_breaker_strategy=failure",
	"--experimental_remote_min_fail_count_to_compute_failure_rate=1",
	"--experimental_remote_min_call_count_to_compute_failure_rate=1",
	"--experimental_remote_failure_rate_threshold=1",
	"--remote_retries=0");
	write("a/bar.in", "updated bar");
	int stderrLenBefore = worker.getStderr().length();
	worker.armReadFailures();

	var error = assertThrows(BuildFailedException.class, () -> buildTarget("//a:bar"));

	// Assert: the breaker tripped and, without recovery, the build failed with a remote error.
	assertReadFailureInjectedSince(stderrLenBefore);
	assertThat(error.getDetailedExitCode().getExitCode().getNumericExitCode()).isEqualTo(34);
	}
	}