PythonUtils.GetInitPyFiles gains a Predicate<PathFragment> supplied at construction time so that other implementations can supply their own logic for determining when a file is a Python package defining __init__.py equivalent. Adds PythonSemantics.getEmptyRunFilesSupplier() and moves the static PythonUtils.GET_INIT_PY_FILES into the BazelPythonSemantics implementation in favor of PyExecutable calling the abstract getEmptyRunfilesSupplier() so that alternate PythonSemantics implementations can supply their own behavior. PiperOrigin-RevId: 291068951
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java index 7beb823..eb8077c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java
@@ -49,18 +49,23 @@ import com.google.devtools.build.lib.rules.python.PyRuntimeInfo; import com.google.devtools.build.lib.rules.python.PythonConfiguration; import com.google.devtools.build.lib.rules.python.PythonSemantics; +import com.google.devtools.build.lib.rules.python.PythonUtils; import com.google.devtools.build.lib.rules.python.PythonVersion; import com.google.devtools.build.lib.util.FileTypeSet; import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.PathFragment; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.function.Predicate; import javax.annotation.Nullable; /** Functionality specific to the Python rules in Bazel. */ public class BazelPythonSemantics implements PythonSemantics { + public static final Runfiles.EmptyFilesSupplier GET_INIT_PY_FILES = + new PythonUtils.GetInitPyFiles((Predicate<PathFragment> & Serializable) source -> false); private static final Template STUB_TEMPLATE = Template.forResource(BazelPythonSemantics.class, "python_stub_template.txt"); public static final InstrumentationSpec PYTHON_COLLECTION_SPEC = new InstrumentationSpec( @@ -70,6 +75,11 @@ public static final PathFragment ZIP_RUNFILES_DIRECTORY_NAME = PathFragment.create("runfiles"); @Override + public Runfiles.EmptyFilesSupplier getEmptyRunfilesSupplier() { + return GET_INIT_PY_FILES; + } + + @Override public String getSrcsVersionDocURL() { // TODO(#8996): Update URL to point to rules_python's docs instead of the Bazel site. return "https://docs.bazel.build/versions/master/be/python.html#py_binary.srcs_version";
diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyExecutable.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyExecutable.java index 3f0a2db..c0dfafe 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/python/PyExecutable.java +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyExecutable.java
@@ -126,7 +126,8 @@ * * <p>See {@link PythonUtils#getInitPyFiles} for details about how the files are created. */ - private static void maybeCreateInitFiles(RuleContext ruleContext, Runfiles.Builder builder) { + private static void maybeCreateInitFiles( + RuleContext ruleContext, Runfiles.Builder builder, PythonSemantics semantics) { boolean createFiles; if (!ruleContext.attributes().has("legacy_create_init", BuildType.TRISTATE)) { createFiles = true; @@ -139,7 +140,7 @@ } } if (createFiles) { - builder.setEmptyFilesSupplier(PythonUtils.GET_INIT_PY_FILES); + builder.setEmptyFilesSupplier(semantics.getEmptyRunfilesSupplier()); } } @@ -157,7 +158,7 @@ semantics.collectDefaultRunfiles(ruleContext, builder); builder.add(ruleContext, PythonRunfilesProvider.TO_RUNFILES); - maybeCreateInitFiles(ruleContext, builder); + maybeCreateInitFiles(ruleContext, builder, semantics); semantics.collectRunfilesForBinary(ruleContext, builder, common, ccInfo); return builder.build();
diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java index 4225a4c..5a055ce 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java
@@ -93,4 +93,12 @@ throws InterruptedException, RuleErrorException; CcInfo buildCcInfoProvider(Iterable<? extends TransitiveInfoCollection> deps); + + /** + * Called when building executables or packages to fill in missing empty __init__.py files if the + * --incompatible_default_to_explicit_init_py has not yet been enabled. This usually returns a + * public static final reference, code is free to use that directly on specific implementations + * instead of making this call. + */ + Runfiles.EmptyFilesSupplier getEmptyRunfilesSupplier(); }
diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonUtils.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonUtils.java index a53ae8f..47843ef 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/python/PythonUtils.java +++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonUtils.java
@@ -22,13 +22,13 @@ import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; import com.google.devtools.build.lib.analysis.actions.SpawnAction; import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; -import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.util.FileType; import com.google.devtools.build.lib.vfs.PathFragment; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; /** * Various utility methods for Python support. @@ -40,48 +40,71 @@ private static final FileType REQUIRES_INIT_PY = FileType.of(".py", ".so", ".pyc"); - static class GetInitPyFiles implements Runfiles.EmptyFilesSupplier { + /** + * Used to get the set of empty __init__.py files to be added to a given set of files to allow the + * Python runtime to import subdirectories potentially containing Python code to be imported as + * packages. Ideally this feature goes away with --incompatible_default_to_explicit_init_py as the + * long term default behavior. + */ + public static class GetInitPyFiles implements Runfiles.EmptyFilesSupplier { + private final Predicate<PathFragment> isPackageInit; + + public GetInitPyFiles(Predicate<PathFragment> isPackageInit) { + this.isPackageInit = isPackageInit; + } + @Override - public Iterable<PathFragment> getExtraPaths(Set<PathFragment> manifestPaths) { + public Set<PathFragment> getExtraPaths(Set<PathFragment> manifestPaths) { return getInitPyFiles(manifestPaths); } - } - @AutoCodec - public static final Runfiles.EmptyFilesSupplier GET_INIT_PY_FILES = new GetInitPyFiles(); + /** + * Returns the set of empty __init__.py(c) files to be added to a given set of files to allow + * the Python runtime to find the <code>.py</code> and <code>.so</code> files present in the + * tree. + */ + private Set<PathFragment> getInitPyFiles(Set<PathFragment> manifestFiles) { + Set<PathFragment> result = new HashSet<>(); + Set<PathFragment> hasPackageInitDirs = new HashSet<>(); // For b/142135992. - private PythonUtils() { - // This is a utility class, not to be instantiated - } + // Find directories containing Python package init files based on a caller supplied test in + // order to support non-standard Python package init naming schemes. + // This loop is done prior to the one below as we assume no order in the set and that we may + // find inits in parent directories listed after subdirectories which the nested loop below + // would need to know of. + for (PathFragment source : manifestFiles) { + if (isPackageInit.test(source)) { + hasPackageInitDirs.add(source.getParentDirectory()); + // Refactoring note... the PathFragment creation and lookups in the loop below could + // be merged into this loop. + } + } - /** - * Returns the set of empty __init__.py(c) files to be added to a given set of files to allow - * the Python runtime to find the <code>.py</code> and <code>.so</code> files present in the - * tree. - */ - public static Set<PathFragment> getInitPyFiles(Set<PathFragment> manifestFiles) { - Set<PathFragment> result = new HashSet<>(); + for (PathFragment source : manifestFiles) { + // If we have a python or .so file at this level... + if (REQUIRES_INIT_PY.matches(source)) { + // ...then record that we need an __init__.py in this and all parents directories... + while (source.segmentCount() > 1) { + source = source.getParentDirectory(); + // ...unless it's a Python .pyc cache or we already have __init__ there. + if (!source.endsWith(PYCACHE) && !hasPackageInitDirs.contains(source)) { + PathFragment initpy = source.getRelative(INIT_PY); + PathFragment initpyc = source.getRelative(INIT_PYC); - for (PathFragment source : manifestFiles) { - // If we have a python or .so file at this level... - if (REQUIRES_INIT_PY.matches(source)) { - // ...then record that we need an __init__.py in this and all parents directories... - while (source.segmentCount() > 1) { - source = source.getParentDirectory(); - // ...unless it's a Python .pyc cache or we already have __init__ there. - if (!source.endsWith(PYCACHE)) { - PathFragment initpy = source.getRelative(INIT_PY); - PathFragment initpyc = source.getRelative(INIT_PYC); - - if (!manifestFiles.contains(initpy) && !manifestFiles.contains(initpyc)) { - result.add(initpy); + if (!manifestFiles.contains(initpy) && !manifestFiles.contains(initpyc)) { + result.add(initpy); + } } } } } - } - return ImmutableSet.copyOf(result); + return ImmutableSet.copyOf(result); + } + } + + private PythonUtils() { + // This is a utility class, not to be instantiated } /**