| # Copyright 2022 The Bazel Authors. All rights reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Various things common to Bazel and Google rule implementations.""" |
| |
| load(":common/cc/cc_helper.bzl", "cc_helper") |
| load( |
| ":common/python/providers.bzl", |
| "PyInfo", |
| ) |
| |
| _platform_common = _builtins.toplevel.platform_common |
| _coverage_common = _builtins.toplevel.coverage_common |
| |
| # Extensions without the dot |
| _PYTHON_SOURCE_EXTENSIONS = ["py"] |
| |
| # Todo: rename this to "binary" and split off library-parts |
| def create_binary_semantics_struct( |
| *, |
| create_executable, |
| get_cc_details_for_binary, |
| get_central_uncachable_version_file, |
| get_coverage_deps, |
| get_debugger_deps, |
| get_extra_common_runfiles_for_binary, |
| get_extra_providers, |
| get_extra_write_build_data_env, |
| get_interpreter_path, |
| get_imports, |
| get_native_deps_dso_name, |
| get_native_deps_user_link_flags, |
| get_stamp_flag, |
| maybe_precompile, |
| should_build_native_deps_dso, |
| should_create_init_files, |
| should_include_build_data): |
| """Helper to ensure a semantics struct has all necessary fields. |
| |
| Call this instead of a raw call to `struct(...)`; it'll help ensure all |
| the necessary functions are being correctly provided. |
| |
| Args: |
| create_executable: Callable; creates a binary's executable output. See |
| py_executable.bzl#py_executable_impl for details. |
| get_cc_details_for_binary: Callable that returns a `CcDetails` struct; see |
| `create_cc_detail_struct`. |
| get_central_uncachable_version_file: Callable that returns an optional |
| Artifact; this artifact is special: it is never cached and is a copy |
| of `ctx.version_file`; see py_builtins.copy_without_caching |
| get_coverage_deps: Callable that returns a list of Targets for making |
| coverage work; only called if coverage is enabled. |
| get_debugger_deps: Callable that returns a list of Targets that provide |
| custom debugger support; only called for target-configuration. |
| get_extra_common_runfiles_for_binary: Callable that returns a runfiles |
| object of extra runfiles a binary should include. |
| get_extra_providers: Callable that returns extra providers; see |
| py_executable.bzl#_create_providers for details. |
| get_extra_write_build_data_env: Callable that returns a dict[str, str] |
| of additional environment variable to pass to build data generation. |
| get_interpreter_path: Callable that returns an optional string, which is |
| the path to the Python interpreter to use for running the binary. |
| get_imports: Callable that returns a depset of the target's import |
| paths. |
| get_native_deps_dso_name: Callable that returns a string, which is the |
| basename (with extension) of the native deps DSO library. |
| get_native_deps_user_link_flags: Callable that returns a list of strings, |
| which are any extra linker flags to pass onto the native deps DSO |
| linking action. |
| get_stamp_flag: Callable that returns bool of if the --stamp flag was |
| enabled or not. |
| maybe_precompile: Callable that may optional precompile the input `.py` |
| sources and returns the full set of desired outputs derived from |
| the source files (e.g., both py and pyc, only one of them, etc). |
| should_build_native_deps_dso: Callable that returns bool; True if |
| building a native deps DSO is supported, False if not. |
| should_create_init_files: Callable that returns bool; True if |
| `__init__.py` files should be generated, False if not. |
| should_include_build_data: Callable that returns bool; True if |
| build data should be generated, False if not. |
| Returns: |
| A "BinarySemantics" struct. |
| """ |
| return struct( |
| # keep-sorted |
| create_executable = create_executable, |
| get_cc_details_for_binary = get_cc_details_for_binary, |
| get_central_uncachable_version_file = get_central_uncachable_version_file, |
| get_coverage_deps = get_coverage_deps, |
| get_debugger_deps = get_debugger_deps, |
| get_extra_common_runfiles_for_binary = get_extra_common_runfiles_for_binary, |
| get_extra_providers = get_extra_providers, |
| get_extra_write_build_data_env = get_extra_write_build_data_env, |
| get_imports = get_imports, |
| get_interpreter_path = get_interpreter_path, |
| get_native_deps_dso_name = get_native_deps_dso_name, |
| get_native_deps_user_link_flags = get_native_deps_user_link_flags, |
| get_stamp_flag = get_stamp_flag, |
| maybe_precompile = maybe_precompile, |
| should_build_native_deps_dso = should_build_native_deps_dso, |
| should_create_init_files = should_create_init_files, |
| should_include_build_data = should_include_build_data, |
| ) |
| |
| def create_library_semantics_struct( |
| *, |
| get_cc_info_for_library, |
| get_imports, |
| maybe_precompile): |
| """Create a `LibrarySemantics` struct. |
| |
| Call this instead of a raw call to `struct(...)`; it'll help ensure all |
| the necessary functions are being correctly provided. |
| |
| Args: |
| get_cc_info_for_library: Callable that returns a CcInfo for the library; |
| see py_library_impl for arg details. |
| get_imports: Callable; see create_binary_semantics_struct. |
| maybe_precompile: Callable; see create_binary_semantics_struct. |
| Returns: |
| a `LibrarySemantics` struct. |
| """ |
| return struct( |
| # keep sorted |
| get_cc_info_for_library = get_cc_info_for_library, |
| get_imports = get_imports, |
| maybe_precompile = maybe_precompile, |
| ) |
| |
| def create_cc_details_struct( |
| *, |
| cc_info_for_propagating, |
| cc_info_for_self_link, |
| cc_info_with_extra_link_time_libraries, |
| extra_runfiles, |
| cc_toolchain): |
| """Creates a CcDetails struct. |
| |
| Args: |
| cc_info_for_propagating: CcInfo that is propagated out of the target |
| by returning it within a PyCcLinkParamsProvider object. |
| cc_info_for_self_link: CcInfo that is used when linking for the |
| binary (or its native deps DSO) itself. This may include extra |
| information that isn't propagating (e.g. a custom malloc) |
| cc_info_with_extra_link_time_libraries: CcInfo of extra link time |
| libraries that MUST come after `cc_info_for_self_link` (or possibly |
| always last; not entirely clear) when passed to |
| `link.linking_contexts`. |
| extra_runfiles: runfiles of extra files needed at runtime, usually as |
| part of `cc_info_with_extra_link_time_libraries`; should be added to |
| runfiles. |
| cc_toolchain: CcToolchain that should be used when building. |
| |
| Returns: |
| A `CcDetails` struct. |
| """ |
| return struct( |
| cc_info_for_propagating = cc_info_for_propagating, |
| cc_info_for_self_link = cc_info_for_self_link, |
| cc_info_with_extra_link_time_libraries = cc_info_with_extra_link_time_libraries, |
| extra_runfiles = extra_runfiles, |
| cc_toolchain = cc_toolchain, |
| ) |
| |
| def union_attrs(*attr_dicts, allow_none = False): |
| """Helper for combining and building attriute dicts for rules. |
| |
| Similar to dict.update, except: |
| * Duplicate keys raise an error if they aren't equal. This is to prevent |
| unintentionally replacing an attribute with a potentially incompatible |
| definition. |
| * None values are special: They mean the attribute is required, but the |
| value should be provided by another attribute dict (depending on the |
| `allow_none` arg). |
| Args: |
| *attr_dicts: The dicts to combine. |
| allow_none: bool, if True, then None values are allowed. If False, |
| then one of `attrs_dicts` must set a non-None value for keys |
| with a None value. |
| |
| Returns: |
| dict of attributes. |
| """ |
| result = {} |
| missing = {} |
| for attr_dict in attr_dicts: |
| for attr_name, value in attr_dict.items(): |
| if value == None and not allow_none: |
| if attr_name not in result: |
| missing[attr_name] = None |
| else: |
| if attr_name in missing: |
| missing.pop(attr_name) |
| |
| if attr_name not in result or result[attr_name] == None: |
| result[attr_name] = value |
| elif value != None and result[attr_name] != value: |
| fail("Duplicate attribute name: '{}': existing={}, new={}".format( |
| attr_name, |
| result[attr_name], |
| value, |
| )) |
| |
| # Else, they're equal, so do nothing. This allows merging dicts |
| # that both define the same key from a common place. |
| |
| if missing and not allow_none: |
| fail("Required attributes missing: " + csv(missing.keys())) |
| return result |
| |
| def csv(values): |
| """Convert a list of strings to comma separated value string.""" |
| return ", ".join(sorted(values)) |
| |
| def filter_to_py_srcs(srcs): |
| """Filters .py files from the given list of files""" |
| |
| # TODO(b/203567235): Get the set of recognized extensions from |
| # elsewhere, as there may be others. e.g. Bazel recognizes .py3 |
| # as a valid extension. |
| return [f for f in srcs if f.extension == "py"] |
| |
| def collect_runfiles(ctx, files): |
| """Collects the necessary files from the rule's context. |
| |
| This presumes the ctx is for a py_binary, py_test, or py_library rule. |
| |
| Args: |
| ctx: rule ctx |
| files: depset of extra files to include in the runfiles. |
| Returns: |
| runfiles necessary for the ctx's target. |
| """ |
| return ctx.runfiles( |
| transitive_files = files, |
| # This little arg carries a lot of weight, but because Starlark doesn't |
| # have a way to identify if a target is just a File, the equivalent |
| # logic can't be re-implemented in pure-Starlark. |
| # |
| # Under the hood, it calls the Java `Runfiles#addRunfiles(ctx, |
| # DEFAULT_RUNFILES)` method, which is the what the Java implementation |
| # of the Python rules originally did, and the details of how that method |
| # works have become relied on in various ways. Specifically, what it |
| # does is visit the srcs, deps, and data attributes in the following |
| # ways: |
| # |
| # For each target in the "data" attribute... |
| # If the target is a File, then add that file to the runfiles. |
| # Otherwise, add the target's **data runfiles** to the runfiles. |
| # |
| # Note that, contray to best practice, the default outputs of the |
| # targets in `data` are *not* added, nor are the default runfiles. |
| # |
| # This ends up being important for several reasons, some of which are |
| # specific to Google-internal features of the rules. |
| # * For Python executables, we have to use `data_runfiles` to avoid |
| # conflicts for the build data files. Such files have |
| # target-specific content, but uses a fixed location, so if a |
| # binary has another binary in `data`, and both try to specify a |
| # file for that file path, then a warning is printed and an |
| # arbitrary one will be used. |
| # * For rules with _entirely_ different sets of files in data runfiles |
| # vs default runfiles vs default outputs. For example, |
| # proto_library: documented behavior of this rule is that putting it |
| # in the `data` attribute will cause the transitive closure of |
| # `.proto` source files to be included. This set of sources is only |
| # in the `data_runfiles` (`default_runfiles` is empty). |
| # * For rules with a _subset_ of files in data runfiles. For example, |
| # a certain Google rule used for packaging arbitrary binaries will |
| # generate multiple versions of a binary (e.g. different archs, |
| # stripped vs un-stripped, etc) in its default outputs, but only |
| # one of them in the runfiles; this helps avoid large, unused |
| # binaries contributing to remote executor input limits. |
| # |
| # Unfortunately, the above behavior also results in surprising behavior |
| # in some cases. For example, simple custom rules that only return their |
| # files in their default outputs won't have their files included. Such |
| # cases must either return their files in runfiles, or use `filegroup()` |
| # which will do so for them. |
| # |
| # For each target in "srcs" and "deps"... |
| # Add the default runfiles of the target to the runfiles. While this |
| # is desirable behavior, it also ends up letting a `py_library` |
| # be put in `srcs` and still mostly work. |
| # TODO(b/224640180): Reject py_library et al rules in srcs. |
| collect_default = True, |
| ) |
| |
| def create_py_info(ctx, *, direct_sources, imports): |
| """Create PyInfo provider. |
| |
| Args: |
| ctx: rule ctx. |
| direct_sources: depset of Files; the direct, raw `.py` sources for the |
| target. This should only be Python source files. It should not |
| include pyc files. |
| imports: depset of strings; the import path values to propagate. |
| |
| Returns: |
| A tuple of the PyInfo instance and a depset of the |
| transitive sources collected from dependencies (the latter is only |
| necessary for deprecated extra actions support). |
| """ |
| uses_shared_libraries = False |
| transitive_sources_depsets = [] # list of depsets |
| transitive_sources_files = [] # list of Files |
| for target in ctx.attr.deps: |
| # PyInfo may not be present for e.g. cc_library rules. |
| if PyInfo in target: |
| info = target[PyInfo] |
| transitive_sources_depsets.append(info.transitive_sources) |
| uses_shared_libraries = uses_shared_libraries or info.uses_shared_libraries |
| else: |
| # TODO(b/228692666): Remove this once non-PyInfo targets are no |
| # longer supported in `deps`. |
| files = target.files.to_list() |
| for f in files: |
| if f.extension == "py": |
| transitive_sources_files.append(f) |
| uses_shared_libraries = ( |
| uses_shared_libraries or |
| cc_helper.is_valid_shared_library_artifact(f) |
| ) |
| deps_transitive_sources = depset( |
| direct = transitive_sources_files, |
| transitive = transitive_sources_depsets, |
| ) |
| |
| # We only look at data to calculate uses_shared_libraries, if it's already |
| # true, then we don't need to waste time looping over it. |
| if not uses_shared_libraries: |
| # Similar to the above, except we only calculate uses_shared_libraries |
| for target in ctx.attr.data: |
| # TODO(b/234730058): Remove checking for PyInfo in data once depot |
| # cleaned up. |
| if PyInfo in target: |
| info = target[PyInfo] |
| uses_shared_libraries = info.uses_shared_libraries |
| else: |
| files = target.files.to_list() |
| for f in files: |
| uses_shared_libraries = cc_helper.is_valid_shared_library_artifact(f) |
| if uses_shared_libraries: |
| break |
| if uses_shared_libraries: |
| break |
| |
| # TODO(b/203567235): Set `uses_shared_libraries` field, though the Bazel |
| # docs indicate it's unused in Bazel and may be removed. |
| py_info = PyInfo( |
| transitive_sources = depset( |
| transitive = [deps_transitive_sources, direct_sources], |
| ), |
| imports = imports, |
| # NOTE: This isn't strictly correct, but with Python 2 gone, |
| # the srcs_version logic is largely defunct, so shouldn't matter in |
| # practice. |
| has_py2_only_sources = False, |
| has_py3_only_sources = False, |
| uses_shared_libraries = uses_shared_libraries, |
| ) |
| return py_info, deps_transitive_sources |
| |
| def create_instrumented_files_info(ctx): |
| return _coverage_common.instrumented_files_info( |
| ctx, |
| source_attributes = ["srcs"], |
| dependency_attributes = ["deps", "data"], |
| extensions = _PYTHON_SOURCE_EXTENSIONS, |
| ) |
| |
| def create_output_group_info(transitive_sources): |
| return OutputGroupInfo( |
| compilation_prerequisites_INTERNAL_ = transitive_sources, |
| compilation_outputs = transitive_sources, |
| ) |
| |
| _BOOL_TYPE = type(True) |
| |
| def is_bool(v): |
| return type(v) == _BOOL_TYPE |
| |
| def target_platform_has_any_constraint(ctx, constraints): |
| """Check if target platform has any of a list of constraints. |
| |
| Args: |
| ctx: rule context. |
| constraints: label_list of constraints. |
| |
| Returns: |
| True if target platform has at least one of the constraints. |
| """ |
| for constraint in constraints: |
| constraint_value = constraint[_platform_common.ConstraintValueInfo] |
| if ctx.target_platform_has_constraint(constraint_value): |
| return True |
| return False |