Record correct exit code for uncaught exceptions in async threads. Async threads are divorced from the server's error-reporting mechanism. In the event of a server shutdown originating in an async-thread, write the exit code to a file that can be read by the client. -- PiperOrigin-RevId: 141920284 MOS_MIGRATED_REVID=141920284
diff --git a/src/main/cpp/BUILD b/src/main/cpp/BUILD index 9b8289e..82ed551 100644 --- a/src/main/cpp/BUILD +++ b/src/main/cpp/BUILD
@@ -60,19 +60,6 @@ ], ) -cc_library( - name = "blaze_abrupt_exit", - srcs = [ - "blaze_abrupt_exit.cc", - ], - hdrs = [ - "blaze_abrupt_exit.h", - ], - deps = [ - "//src/main/cpp/util:blaze_exit_code", - ], -) - cc_binary( name = "client", srcs = [ @@ -109,7 +96,6 @@ }), visibility = ["//src:__pkg__"], deps = [ - ":blaze_abrupt_exit", ":blaze_util", "//src/main/cpp/util", "//src/main/cpp/util:logging",
diff --git a/src/main/cpp/blaze.cc b/src/main/cpp/blaze.cc index d3b039e..07057b9 100644 --- a/src/main/cpp/blaze.cc +++ b/src/main/cpp/blaze.cc
@@ -54,7 +54,6 @@ #include <vector> -#include "src/main/cpp/blaze_abrupt_exit.h" #include "src/main/cpp/blaze_util.h" #include "src/main/cpp/blaze_util_platform.h" #include "src/main/cpp/global_variables.h" @@ -484,6 +483,11 @@ } else { result.push_back("--client_debug=false"); } + if (globals->options->use_custom_exit_code_on_abrupt_exit) { + result.push_back("--use_custom_exit_code_on_abrupt_exit=true"); + } else { + result.push_back("--use_custom_exit_code_on_abrupt_exit=false"); + } // This is only for Blaze reporting purposes; the real interpretation of the // jvm flags occurs when we set up the java command line. @@ -1271,10 +1275,36 @@ } } -// TODO(bazel-team): Execute the server as a child process and write its exit -// code to a file. In case the server becomes unresonsive or terminates -// unexpectedly (in a way that isn't already handled), we can observe the file, -// if it exists. (If it doesn't, then we know something went horribly wrong.) +int GetExitCodeForAbruptExit(const GlobalVariables &globals) { + const StartupOptions *startup_options = globals.options; + if (startup_options->use_custom_exit_code_on_abrupt_exit) { + BAZEL_LOG(INFO) << "Looking for a custom exit-code."; + std::string filename = blaze_util::JoinPath( + globals.options->output_base, "exit_code_to_use_on_abrupt_exit"); + std::string content; + if (!blaze_util::ReadFile(filename, &content)) { + BAZEL_LOG(INFO) << "Unable to read the custom exit-code file. " + << "Exiting with an INTERNAL_ERROR."; + return blaze_exit_code::INTERNAL_ERROR; + } + if (!blaze_util::UnlinkPath(filename)) { + BAZEL_LOG(INFO) << "Unable to delete the custom exit-code file. " + << "Exiting with an INTERNAL_ERROR."; + return blaze_exit_code::INTERNAL_ERROR; + } + int custom_exit_code; + if (!blaze_util::safe_strto32(content, &custom_exit_code)) { + BAZEL_LOG(INFO) << "Content of custom exit-code file not an int: " + << content << "Exiting with an INTERNAL_ERROR."; + return blaze_exit_code::INTERNAL_ERROR; + } + BAZEL_LOG(INFO) << "Read exit code " << custom_exit_code + << " from custom exit-code file. Exiting accordingly."; + return custom_exit_code; + } + return blaze_exit_code::INTERNAL_ERROR; +} + int Main(int argc, const char *argv[], WorkspaceLayout* workspace_layout, OptionProcessor *option_processor, std::unique_ptr<blaze_util::LogHandler> log_handler) {
diff --git a/src/main/cpp/blaze_abrupt_exit.cc b/src/main/cpp/blaze_abrupt_exit.cc deleted file mode 100644 index e0b5b17..0000000 --- a/src/main/cpp/blaze_abrupt_exit.cc +++ /dev/null
@@ -1,23 +0,0 @@ -// Copyright 2016 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "src/main/cpp/blaze_abrupt_exit.h" -#include "src/main/cpp/util/exit_code.h" - -namespace blaze { - -int GetExitCodeForAbruptExit(const GlobalVariables& globals) { - return blaze_exit_code::INTERNAL_ERROR; -} - -} // namespace blaze
diff --git a/src/main/cpp/blaze_abrupt_exit.h b/src/main/cpp/blaze_abrupt_exit.h deleted file mode 100644 index 0e9c22f..0000000 --- a/src/main/cpp/blaze_abrupt_exit.h +++ /dev/null
@@ -1,30 +0,0 @@ -// Copyright 2016 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// -// blaze_abrupt_exit.h: Deals with abrupt exits of the Blaze server. -// -#ifndef BAZEL_SRC_MAIN_CPP_BLAZE_ABRUPT_EXIT_H_ -#define BAZEL_SRC_MAIN_CPP_BLAZE_ABRUPT_EXIT_H_ - -namespace blaze { - -struct GlobalVariables; - -// Returns the exit code to use for when the Blaze server exits abruptly. -int GetExitCodeForAbruptExit(const GlobalVariables& globals); - -} // namespace blaze - -#endif // BAZEL_SRC_MAIN_CPP_BLAZE_ABRUPT_EXIT_H_
diff --git a/src/main/cpp/startup_options.cc b/src/main/cpp/startup_options.cc index d16eeef..ea92cff 100644 --- a/src/main/cpp/startup_options.cc +++ b/src/main/cpp/startup_options.cc
@@ -55,7 +55,8 @@ command_port(0), connect_timeout_secs(10), invocation_policy(NULL), - client_debug(false) { + client_debug(false), + use_custom_exit_code_on_abrupt_exit(true) { bool testing = !blaze::GetEnv("TEST_TMPDIR").empty(); if (testing) { output_root = MakeAbsolute(blaze::GetEnv("TEST_TMPDIR")); @@ -68,11 +69,20 @@ output_root, "_" + product_name_lower + "_" + GetUserName()); // 3 hours (but only 15 seconds if used within a test) max_idle_secs = testing ? 15 : (3 * 3600); - nullary_options = {"deep_execroot", "block_for_lock", - "host_jvm_debug", "master_blazerc", "master_bazelrc", "batch", - "batch_cpu_scheduling", "allow_configurable_attributes", - "fatal_event_bus_exceptions", "experimental_oom_more_eagerly", - "write_command_log", "watchfs", "client_debug"}; + nullary_options = {"deep_execroot", + "block_for_lock", + "host_jvm_debug", + "master_blazerc", + "master_bazelrc", + "batch", + "batch_cpu_scheduling", + "allow_configurable_attributes", + "fatal_event_bus_exceptions", + "experimental_oom_more_eagerly", + "write_command_log", + "watchfs", + "client_debug", + "use_custom_exit_code_on_abrupt_exit"}; unary_options = {"output_base", "install_base", "output_user_root", "host_jvm_profile", "host_javabase", "host_jvm_args", "bazelrc", "blazerc", "io_nice_level", @@ -266,6 +276,12 @@ } else if (GetNullaryOption(arg, "--noclient_debug")) { client_debug = false; option_sources["client_debug"] = rcfile; + } else if (GetNullaryOption(arg, "--use_custom_exit_code_on_abrupt_exit")) { + use_custom_exit_code_on_abrupt_exit = true; + option_sources["use_custom_exit_code_on_abrupt_exit"] = rcfile; + } else if (GetNullaryOption(arg, "--nouse_custom_exit_code_on_abrupt_exit")) { + use_custom_exit_code_on_abrupt_exit = false; + option_sources["use_custom_exit_code_on_abrupt_exit"] = rcfile; } else if ((value = GetUnaryOption( arg, next_arg, "--connect_timeout_secs")) != NULL) { if (!blaze_util::safe_strto32(value, &connect_timeout_secs) ||
diff --git a/src/main/cpp/startup_options.h b/src/main/cpp/startup_options.h index 7020c04..b36a70a 100644 --- a/src/main/cpp/startup_options.h +++ b/src/main/cpp/startup_options.h
@@ -208,6 +208,10 @@ // Whether to output addition debugging information in the client. bool client_debug; + // Whether to check custom file for exit code when the Blaze Server exits + // abruptly without proper communication over gRPC. + bool use_custom_exit_code_on_abrupt_exit; + protected: // Constructor for subclasses only so that site-specific extensions of this // class can override the product name. The product_name must be the
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java index 999d286..74c49f4 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
@@ -58,6 +58,7 @@ import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.BlazeClock; import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.CustomExitCodePublisher; import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.LoggingUtil; import com.google.devtools.build.lib.util.OS; @@ -983,7 +984,9 @@ ExitCode.LOCAL_ENVIRONMENTAL_ERROR); } runtime.initWorkspace(directories, binTools); - + if (startupOptions.useCustomExitCodeOnAbruptExit) { + CustomExitCodePublisher.setAbruptExitStatusFileDir(serverDirectories.getOutputBase()); + } AutoProfiler.setClock(runtime.getClock()); BugReport.setRuntime(runtime); return runtime;
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java index 64a9da7..fe4fe8c 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java
@@ -286,4 +286,16 @@ category = "server startup", help = "The amount of time the client waits for each attempt to connect to the server") public int connectTimeoutSecs; + + @Option( + name = "use_custom_exit_code_on_abrupt_exit", + defaultValue = "true", // NOTE: purely decorative! + category = "undocumented", + help = + "If the Blaze server exits abruptly without access to traditional channels to " + + "communicate its exit code, attempt to communicate through a backup channel. " + + "This flag is temporary to allow easy disabling of a new behavior. " + + "Please do not use this flag." + ) + public boolean useCustomExitCodeOnAbruptExit; }
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java index 9a6e978..d047036 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java
@@ -17,6 +17,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.util.CustomExitCodePublisher; import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.LoggingUtil; import com.google.devtools.build.lib.util.Preconditions; @@ -94,6 +95,7 @@ // to do as a best-effort operation. runtime.shutdownOnCrash(); } + CustomExitCodePublisher.maybeWriteExitStatusFile(exitCode); } finally { // Avoid shutdown deadlock issues: If an application shutdown hook crashes, it will // trigger our Blaze crash handler (this method). Calling System#exit() here, would
diff --git a/src/main/java/com/google/devtools/build/lib/util/CustomExitCodePublisher.java b/src/main/java/com/google/devtools/build/lib/util/CustomExitCodePublisher.java new file mode 100644 index 0000000..c281f5a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/CustomExitCodePublisher.java
@@ -0,0 +1,45 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import java.io.IOException; +import javax.annotation.Nullable; + +/** + * Provides an external way for the Bazel server to communicate its exit code to the client, when + * the main gRPC channel is unavailable because the exit is too abrupt or originated in an async + * thread. + */ +public class CustomExitCodePublisher { + private static final String EXIT_CODE_FILENAME = "exit_code_to_use_on_abrupt_exit"; + @Nullable private static Path abruptExitCodeFilePath = null; + + public static void setAbruptExitStatusFileDir(Path path) { + abruptExitCodeFilePath = path.getChild(EXIT_CODE_FILENAME); + } + + public static void maybeWriteExitStatusFile(int exitCode) { + if (abruptExitCodeFilePath != null) { + try { + FileSystemUtils.writeContentAsLatin1(abruptExitCodeFilePath, String.valueOf(exitCode)); + } catch (IOException ioe) { + System.err.printf( + "io error writing %d to abrupt exit status file %s: %s\n", + exitCode, abruptExitCodeFilePath, ioe.getMessage()); + } + } + } +}