blob: b3e33e1d309ab8fcf40b7fbc7790a307377a6b78 [file] [log] [blame]
# Copyright 2025 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# LINT.IfChange(forked_exports)
"""
ThinLTO expands the traditional 2 step compile (N x compile .cc, 1x link (N .o files) into a 4
step process:
1. Bitcode generation (N times). This is produces intermediate LLVM bitcode from a source
file. For this product, it reuses the .o extension.
2. Indexing (once on N files). This takes all bitcode .o files, and for each .o file, it
decides from which other .o files symbols can be inlined. In addition, it generates an
index for looking up these symbols, and an imports file for identifying new input files for
each step 3 {@link LtoBackendAction}.
3. Backend compile (N times). This is the traditional compilation, and uses the same
command line as the Bitcode generation in 1). Since the compiler has many bit code files
available, it can inline functions and propagate constants across .o files. This step is
costly, as it will do traditional optimization. The result is a .lto.o file, a traditional
ELF object file.
4. Backend link (once). This is the traditional link, and produces the final executable.
"""
load("@bazel_skylib//lib:paths.bzl", "paths")
load("//cc/common:cc_helper_internal.bzl", "should_create_per_object_debug_info")
load("//cc/private:cc_internal.bzl", _cc_internal = "cc_internal")
load("//cc/private/rules_impl:native.bzl", _cc_common_internal = "native_cc_common")
LtoBackendArtifactsInfo = provider(
doc = "LtoBackendArtifacts represents a set of artifacts for a single ThinLTO backend compile.",
fields = {
"index": "(None|File) A file containing mapping of symbol => bitcode file containing the " +
" symbol. It will be None when this is a shared non-lto backend.",
"imports": "(None|File) A file containing a list of bitcode files necessary to run " +
"the backend step. It will be None when this is a shared non-lto backend.",
"_bitcode_file": "(File) The bitcode file which is the input of the compile.",
"_object_file": "(File) The result of executing the above command line, an ELF object " +
" file.",
"_dwo_file": "(File) The corresponding dwo file if fission is used.",
},
)
def create_lto_backends(
actions,
lto_compilation_context,
feature_configuration,
cc_toolchain,
use_pic,
object_file_inputs,
lto_output_root_prefix,
lto_obj_root_prefix,
static_libraries_to_link,
allow_lto_indexing,
include_link_static_in_lto_indexing,
prefer_pic_libs):
"""Create the LTO backends for a link.
Args:
actions: (actions) The actions object.
lto_compilation_context: (LtoCompilationContext) The LTO compilation context.
feature_configuration: (feature_configuration) The feature configuration.
cc_toolchain: (CcToolchainInfo) The C++ toolchain.
use_pic: (bool) Whether to use PIC.
object_file_inputs: (depset[File]) The object file inputs.
lto_output_root_prefix: (str) The root prefix for the LTO output files.
lto_obj_root_prefix: (str) The root prefix for the LTO object files.
static_libraries_to_link: (list[LibraryToLink]) The static libraries to link.
allow_lto_indexing: (bool) Whether LTO indexing is allowed.
include_link_static_in_lto_indexing: (bool) Whether to include the static libraries in the
LTO indexing.
prefer_pic_libs: (bool) Whether to prefer PIC static libraries.
Returns:
(list[LtoBackendArtifactsInfo]) The LTO backends.
"""
cpp_config = cc_toolchain._cpp_configuration
debug = should_create_per_object_debug_info(feature_configuration, cpp_config)
compiled = set()
static_library_files = set()
for lib in static_libraries_to_link:
pic = (prefer_pic_libs and lib.pic_static_library != None) or \
lib.static_library == None
library_file = lib.pic_static_library if pic else lib.static_library
if library_file in static_library_files:
# Duplicated static libraries are linked just once and don't error out.
# TODO(b/413333884): Clean up violations and error out
continue
static_library_files.add(library_file)
context = lib._pic_lto_compilation_context if pic else lib._lto_compilation_context
if context:
compiled.update(context.lto_bitcode_inputs.keys())
all_bitcode = []
# Since this link includes object files from another library, we know that library must be
# statically linked, so we need to look at includeLinkStaticInLtoIndexing to decide whether
# to include its objects in the LTO indexing for this target.
if include_link_static_in_lto_indexing:
for lib in static_libraries_to_link:
if not lib._contains_objects:
continue
pic = (prefer_pic_libs and lib.pic_static_library != None) or \
lib.static_library == None
objects = lib.pic_objects if pic else lib.objects
for obj in objects:
if obj in compiled:
all_bitcode.append(obj)
for obj in object_file_inputs:
if obj in lto_compilation_context.lto_bitcode_inputs:
all_bitcode.append(obj)
if lto_output_root_prefix == lto_obj_root_prefix:
for file in all_bitcode:
if file.is_directory:
fail("Thinlto with tree artifacts requires feature use_lto_native_object_directory.")
# Make this a NestedSet to return from LtoBackendAction.getAllowedDerivedInputs. For M binaries
# and N .o files, this is O(M*N). If we had nested sets of bitcode files, it would be O(M + N).
all_bitcode_depset = depset(all_bitcode)
lto_outputs = []
for lib in static_libraries_to_link:
if not lib._contains_objects:
continue
pic = (prefer_pic_libs and lib.pic_static_library != None) or \
lib.static_library == None
objects = lib.pic_objects if pic else lib.objects
lib_lto_compilation_context = lib._pic_lto_compilation_context if pic else lib._lto_compilation_context
shared_lto_backends = lib._pic_shared_non_lto_backends if pic else lib._shared_non_lto_backends
for obj in objects:
if obj not in compiled:
continue
if include_link_static_in_lto_indexing:
backend_user_compile_flags = _backend_user_compile_flags(cpp_config, obj, lib_lto_compilation_context)
lto_outputs.append(create_lto_backend_artifacts(
actions = actions,
lto_output_root_prefix = lto_output_root_prefix,
lto_obj_root_prefix = lto_obj_root_prefix,
bitcode_file = obj,
all_bitcode_files = all_bitcode_depset,
feature_configuration = feature_configuration,
cc_toolchain = cc_toolchain,
use_pic = use_pic,
should_create_per_object_debug_info = debug,
argv = backend_user_compile_flags,
))
else:
if not shared_lto_backends:
fail(("Statically linked test target requires non-LTO backends for its library inputs," +
" but library input %s does not specify shared_non_lto_backends") % lib)
lto_outputs.append(shared_lto_backends[obj])
for obj in object_file_inputs:
if obj not in lto_compilation_context.lto_bitcode_inputs:
continue
backend_user_compile_flags = _backend_user_compile_flags(cpp_config, obj, lto_compilation_context)
if not allow_lto_indexing:
# Depending on whether LTO indexing is allowed, generate an LTO backend
# that will be fed the results of the indexing step, or a dummy LTO backend
# that simply compiles the bitcode into native code without any index-based
# cross module optimization.
actions = _cc_internal.wrap_link_actions(actions, None, True)
lto_outputs.append(create_lto_backend_artifacts(
actions = actions,
lto_output_root_prefix = lto_output_root_prefix,
lto_obj_root_prefix = lto_obj_root_prefix,
bitcode_file = obj,
all_bitcode_files = all_bitcode_depset if allow_lto_indexing else None,
feature_configuration = feature_configuration,
cc_toolchain = cc_toolchain,
use_pic = use_pic,
should_create_per_object_debug_info = debug,
argv = backend_user_compile_flags,
))
return lto_outputs
def _backend_user_compile_flags(cpp_config, obj, context):
argv = []
lto_bitcode_files = context.lto_bitcode_inputs
if obj in lto_bitcode_files:
argv.extend(lto_bitcode_files[obj].copts)
argv.extend(cpp_config.lto_backend_options)
argv.extend(_cc_internal.collect_per_file_lto_backend_opts(cpp_config, obj))
return argv
def create_shared_non_lto_artifacts(
actions,
lto_compilation_context,
is_linker,
feature_configuration,
cc_toolchain,
use_pic,
object_file_inputs):
"""Create the shared non-LTO artifacts for a statically linked library.
Args:
actions: (actions) The actions object.
lto_compilation_context: (LtoCompilationContext) The LTO compilation context.
is_linker: (bool) Whether the link is a linker.
feature_configuration: (feature_configuration) The feature configuration.
cc_toolchain: (CcToolchainInfo) The C++ toolchain.
use_pic: (bool) Whether to use PIC.
object_file_inputs: (depset[File]) The object file inputs.
Returns:
(dict[File, LtoBackendArtifactsInfo]) The shared non-LTO artifacts.
"""
# Only create the shared LTO artifacts for a statically linked library that has bitcode files.
if not lto_compilation_context or is_linker:
return {}
lto_output_root_prefix = "shared.nonlto"
if feature_configuration.is_enabled("use_lto_native_object_directory"):
lto_obj_root_prefix = "shared.nonlto-obj"
else:
lto_obj_root_prefix = "shared.nonlto"
cpp_config = cc_toolchain._cpp_configuration
debug = should_create_per_object_debug_info(feature_configuration, cpp_config)
shared_non_lto_backends = {}
for obj in object_file_inputs:
if obj not in lto_compilation_context.lto_bitcode_inputs:
continue
backend_user_compile_flags = _backend_user_compile_flags(cpp_config, obj, lto_compilation_context)
shared_non_lto_backends[obj] = create_lto_backend_artifacts(
actions = _cc_internal.wrap_link_actions(actions, None, True),
lto_output_root_prefix = lto_output_root_prefix,
lto_obj_root_prefix = lto_obj_root_prefix,
bitcode_file = obj,
all_bitcode_files = None,
feature_configuration = feature_configuration,
cc_toolchain = cc_toolchain,
use_pic = use_pic,
should_create_per_object_debug_info = debug,
argv = backend_user_compile_flags,
)
return shared_non_lto_backends
def create_lto_backend_artifacts(
*,
actions,
lto_output_root_prefix,
lto_obj_root_prefix,
bitcode_file,
all_bitcode_files = None,
feature_configuration,
cc_toolchain,
use_pic,
should_create_per_object_debug_info,
argv):
"""Create an LTO backend.
It uses the appropriate constructor depending on whether the associated
ThinLTO link will utilize LTO indexing (therefore unique LTO backend actions), or not (and
therefore the library being linked will create a set of shared LTO backends).
TODO(b/128341904): Do cross module optimization once there is Starlark support.
If all_bitcode_files is null, create an LTO backend that does not perform any cross-module
optimization, by not generating import and index files.
Args:
actions: (actions) The actions object.
lto_output_root_prefix: (str) The root prefix for the LTO output files.
lto_obj_root_prefix: (str) The root prefix for the LTO object files.
bitcode_file: (File) The bitcode file to create the LTO backend for.
all_bitcode_files: (None|depset[File]) The set of all bitcode files to be indexed.
feature_configuration: (feature_configuration) The feature configuration.
cc_toolchain: (CcToolchainInfo) The C++ toolchain.
use_pic: (bool) Whether to use PIC.
should_create_per_object_debug_info: (bool) Whether to create per-object debug info.
argv: (list[str]) The command line arguments to pass to the LTO backend.
Returns:
An LtoBackendArtifactsInfo provider.
"""
create_shared_non_lto = all_bitcode_files == None
build_variables = {}
additional_inputs = []
_initialize_build_variables(
build_variables,
additional_inputs,
cc_toolchain,
feature_configuration,
argv,
)
build_variables = _cc_internal.cc_toolchain_variables(vars = build_variables)
build_variables = _cc_internal.combine_cc_toolchain_variables(cc_toolchain._build_variables, build_variables)
env = _cc_common_internal.get_environment_variables(
feature_configuration = feature_configuration,
action_name = "lto-backend",
variables = build_variables,
)
obj = lto_obj_root_prefix + "/" + bitcode_file.path
# index_obj is an object that does not exist but helps us find where to store the index and
# imports files
index_obj = lto_output_root_prefix + "/" + bitcode_file.path
additional_inputs = depset(additional_inputs, transitive = [cc_toolchain._compiler_files])
imports, index, dwo_file = None, None, None
if bitcode_file.is_directory:
# declare_shareable_directory is needed to create TreeArtifact in a different configuration (for Android split configurations)
object_file = actions.declare_shareable_directory(obj)
if not create_shared_non_lto:
imports = actions.declare_shareable_directory(index_obj)
index = imports
# No support for dwo files for tree artifacts at the moment. This should not throw an
# irrecoverable exception because we can still generate dwo files for the other artifacts.
# TODO(b/289089713): Add support for dwo files for tree artifacts.
_cc_internal.create_lto_backend_action_template(
actions = actions,
feature_configuration = feature_configuration,
additional_inputs = additional_inputs,
env = env,
build_variables = build_variables,
use_pic = use_pic,
all_bitcode_files = all_bitcode_files,
index = index,
bitcode_file = bitcode_file,
object_file = object_file,
dwo_file = dwo_file,
)
else:
object_file = actions.declare_shareable_artifact(obj)
if not create_shared_non_lto:
imports = actions.declare_shareable_artifact(index_obj + ".imports")
index = actions.declare_shareable_artifact(index_obj + ".thinlto.bc")
if should_create_per_object_debug_info:
dwo_file = actions.declare_shareable_artifact(paths.replace_extension(obj, ".dwo"))
_create_lto_backend_action(
actions,
additional_inputs,
env,
build_variables,
feature_configuration,
index,
imports,
bitcode_file,
object_file,
all_bitcode_files,
dwo_file,
use_pic,
None, # bitcode_file_path
)
return LtoBackendArtifactsInfo(
index = index,
imports = imports,
_bitcode_file = bitcode_file,
_object_file = object_file,
_dwo_file = dwo_file,
)
def _initialize_build_variables(build_variables, additional_inputs, cc_toolchain, feature_configuration, user_compile_flags):
"""
Populates build_variables and additional_inputs with data that is independent of what file is the input to the action.
"""
_add_profile_for_lto_backend(
additional_inputs,
cc_toolchain._fdo_context,
feature_configuration,
build_variables,
)
# Add the context sensitive instrument path to the backend.
if feature_configuration.is_enabled("cs_fdo_instrument"):
build_variables["cs_fdo_instrument_path"] = cc_toolchain._cpp_configuration.cs_fdo_instrument()
build_variables["user_compile_flags"] = _cc_internal.intern_string_sequence_variable_value(user_compile_flags)
if not _cc_common_internal.action_is_enabled(feature_configuration = feature_configuration, action_name = "lto-backend"):
fail("Thinlto build is requested, but the C++ toolchain doesn't define an action_config for 'lto-backend' action.")
def _add_profile_for_lto_backend(additional_inputs, fdo_context, feature_configuration, build_variables):
prefetch = getattr(fdo_context, "prefetch_hints_artifact", None)
if prefetch != None:
build_variables["fdo_prefetch_hints_path"] = prefetch.path
additional_inputs.append(fdo_context.prefetch_hints_artifact)
propeller_optimize_info = getattr(fdo_context, "propeller_optimize_info", None)
if propeller_optimize_info != None and propeller_optimize_info.cc_profile != None:
build_variables["propeller_optimize_cc_path"] = propeller_optimize_info.cc_profile
additional_inputs.append(propeller_optimize_info.cc_profile)
if propeller_optimize_info != None and propeller_optimize_info.ld_profile != None:
build_variables["propeller_optimize_ld_path"] = propeller_optimize_info.ld_profile
additional_inputs.append(propeller_optimize_info.ld_profile)
if not feature_configuration.is_enabled("autofdo") and \
not feature_configuration.is_enabled("cs_fdo_optimize") and \
not feature_configuration.is_enabled("xbinaryfdo"):
return
branch_fdo_profile = getattr(fdo_context, "branch_fdo_profile", None)
if branch_fdo_profile == None:
fail("Branch FDO profile is None")
profile = branch_fdo_profile.profile_artifact
build_variables["fdo_profile_path"] = profile
additional_inputs.append(profile)
# LINT.IfChange(lto_backends)
def _create_lto_backend_action(
actions,
additional_inputs,
env,
build_variables,
feature_configuration,
index,
imports,
bitcode_artifact,
object_file,
bitcode_files,
dwo_file,
use_pic,
bitcode_file_path):
if index != None and index.is_directory:
fail("index cannot be a TreeArtifact")
if imports != None and imports.is_directory:
fail("imports cannot be a TreeArtifact")
if dwo_file != None and dwo_file.is_directory:
fail("dwo_file cannot be a TreeArtifact")
if object_file.is_directory:
fail("object_file cannot be a TreeArtifact")
if bitcode_artifact.is_directory and bitcode_file_path == None:
fail("If bitcode file is a tree artifact, the bitcode file path must contain the path.")
if not bitcode_artifact.is_directory and bitcode_file_path != None:
fail("If bitcode file is not a tree artifact, then bitcode file path should be null to not override the path.")
inputs = _get_lto_backend_action_inputs(index, imports, bitcode_artifact, additional_inputs)
outputs = _get_lto_backend_action_outputs(object_file, dwo_file)
_path_variables = _paths_build_variables(
index,
object_file,
dwo_file,
bitcode_file_path if bitcode_file_path != None else bitcode_artifact,
)
_path_variables = _cc_internal.cc_toolchain_variables(vars = _path_variables)
build_variables = _cc_internal.combine_cc_toolchain_variables(build_variables, _path_variables)
_cc_internal.create_lto_backend_action(
actions = actions,
feature_configuration = feature_configuration,
build_variables = build_variables,
use_pic = use_pic,
inputs = inputs,
all_bitcode_files = bitcode_files,
imports = imports,
outputs = outputs,
env = env,
)
def _paths_build_variables(index, object_file, dwo_file, bitcode_file):
build_variables = {}
# Ideally, those strings would come directly from the execPath of the Artifacts of
# the LtoBackendAction.Builder; however, in order to support tree artifacts, we need
# the bitcode_file_path to be different from the bitcode_tree_artifact execPath.
# The former is a file path and the latter is the directory path.
# Therefore we accept strings as inputs rather than artifacts.
if index != None:
build_variables["thinlto_index"] = index
else:
# An empty input indicates not to perform cross-module optimization.
build_variables["thinlto_index"] = "/dev/null"
# The output from the LTO backend step is a native object file.
build_variables["thinlto_output_object_file"] = object_file
# The input to the LTO backend step is the bitcode file.
build_variables["thinlto_input_bitcode_file"] = bitcode_file
if dwo_file != None:
build_variables["per_object_debug_info_file"] = dwo_file
build_variables["is_using_fission"] = ""
return build_variables
def _get_lto_backend_action_inputs(index, imports, bitcode_file, additional_inputs):
inputs = [bitcode_file]
if imports != None:
# Although the imports file is not used by the LTOBackendAction while the action is
# executing, it is needed during the input discovery phase, and we must list it as an input
# to the action in order for it to be preserved under --discard_orphaned_artifacts.
inputs.append(imports)
if index != None:
inputs.append(index)
return depset(inputs, transitive = [additional_inputs])
def _get_lto_backend_action_outputs(object_file, dwo_file):
outputs = [object_file]
if dwo_file != None:
outputs.append(dwo_file)
return outputs
# LINT.ThenChange(https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/cpp/LtoBackendArtifacts.java:lto_backends)
# LINT.ThenChange(https://github.com/bazelbuild/bazel/blob/master/src/main/starlark/builtins_bzl/common/cc/link/lto_backends.bzl:forked_exports)