Add imports attribute to Bazel native Python rules to allow adding directories to PYTHONPATH.
Fixes #702
RELNOTES: Add imports attribute to native Python rules.
--
MOS_MIGRATED_REVID=114314430
diff --git a/examples/py_native/BUILD b/examples/py_native/BUILD
index d6fdfa9..5a82223 100644
--- a/examples/py_native/BUILD
+++ b/examples/py_native/BUILD
@@ -7,7 +7,10 @@
py_binary(
name = "bin",
srcs = ["bin.py"],
- deps = [":lib"],
+ deps = [
+ ":lib",
+ "//examples/py_native/fibonacci",
+ ],
)
py_library(
@@ -18,7 +21,10 @@
py_test(
name = "test",
srcs = ["test.py"],
- deps = [":lib"],
+ deps = [
+ ":lib",
+ "//examples/py_native/fibonacci",
+ ],
)
py_test(
diff --git a/examples/py_native/bin.py b/examples/py_native/bin.py
index f79379a..7b65627 100644
--- a/examples/py_native/bin.py
+++ b/examples/py_native/bin.py
@@ -1,4 +1,6 @@
"""A tiny example binary for the native Python rules of Bazel."""
from examples.py_native.lib import GetNumber
+from fib import Fib
print "The number is %d" % GetNumber()
+print "Fib(5) == %d" % Fib(5)
diff --git a/examples/py_native/fibonacci/BUILD b/examples/py_native/fibonacci/BUILD
new file mode 100644
index 0000000..c3ee9b0
--- /dev/null
+++ b/examples/py_native/fibonacci/BUILD
@@ -0,0 +1,6 @@
+py_library(
+ name = "fibonacci",
+ srcs = ["fib.py"],
+ imports = ["."],
+ visibility = ["//examples/py_native:__pkg__"],
+)
diff --git a/examples/py_native/fibonacci/fib.py b/examples/py_native/fibonacci/fib.py
new file mode 100644
index 0000000..645a937
--- /dev/null
+++ b/examples/py_native/fibonacci/fib.py
@@ -0,0 +1,8 @@
+"""An example binary to test the imports attribute of native Python rules."""
+
+
+def Fib(n):
+ if n == 0 or n == 1:
+ return 1
+ else:
+ return Fib(n-1) + Fib(n-2)
diff --git a/examples/py_native/test.py b/examples/py_native/test.py
index 811eee1..f9543aa 100644
--- a/examples/py_native/test.py
+++ b/examples/py_native/test.py
@@ -1,6 +1,8 @@
"""A tiny example binary for the native Python rules of Bazel."""
+
import unittest
from examples.py_native.lib import GetNumber
+from fib import Fib
class TestGetNumber(unittest.TestCase):
@@ -8,6 +10,8 @@
def test_ok(self):
self.assertEquals(GetNumber(), 42)
+ def test_fib(self):
+ self.assertEquals(Fib(5), 8)
if __name__ == '__main__':
unittest.main()
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java
index 36d3a1f..7c0fc71 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPyRuleClasses.java
@@ -20,7 +20,9 @@
import static com.google.devtools.build.lib.packages.BuildType.LABEL_LIST;
import static com.google.devtools.build.lib.packages.BuildType.TRISTATE;
import static com.google.devtools.build.lib.syntax.Type.STRING;
+import static com.google.devtools.build.lib.syntax.Type.STRING_LIST;
+import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.analysis.BaseRuleClasses;
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
@@ -63,6 +65,20 @@
.override(builder.copy("deps")
.allowedRuleClasses(ALLOWED_RULES_IN_DEPS)
.allowedFileTypes())
+ /* <!-- #BLAZE_RULE($base_py).ATTRIBUTE(imports) -->
+ List of import directories to be added to the <code>PYTHONPATH</code>.
+ <p>
+ Subject to <a href="make-variables.html">"Make variable"</a> substitution. These import
+ directories will be added for this rule and all rules that depend on it (note: not the
+ rules this rule depends on. Each directory will be added to <code>PYTHONPATH</code> by
+ <a href="#py_binary"><code>py_binary</code></a> rules that depend on this rule.
+ </p>
+ <p>
+ Absolute paths (paths that start with <code>/</code>) and paths that references a path
+ above the execution root are not allowed and will result in an error.
+ </p>
+ <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+ .add(attr("imports", STRING_LIST).value(ImmutableList.<String>of()))
/* <!-- #BLAZE_RULE($base_py).ATTRIBUTE(srcs_version) -->
A string specifying the Python major version(s) that the <code>.py</code> source
files listed in the <code>srcs</code> of this rule are compatible with.
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java
index abff076..7092e5f 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java
@@ -14,6 +14,7 @@
package com.google.devtools.build.lib.bazel.rules.python;
+import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.analysis.RuleContext;
@@ -22,13 +23,18 @@
import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction;
import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Substitution;
import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Template;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore;
import com.google.devtools.build.lib.rules.python.PyCommon;
import com.google.devtools.build.lib.rules.python.PythonSemantics;
import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec;
+import com.google.devtools.build.lib.syntax.Type;
import com.google.devtools.build.lib.util.FileTypeSet;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
/**
* Functionality specific to the Python rules in Bazel.
@@ -64,8 +70,29 @@
}
@Override
+ public List<PathFragment> getImports(RuleContext ruleContext) {
+ List<PathFragment> result = new ArrayList<>();
+ PathFragment packageFragment = ruleContext.getLabel().getPackageIdentifier().getPathFragment();
+ for (String importsAttr : ruleContext.attributes().get("imports", Type.STRING_LIST)) {
+ importsAttr = ruleContext.expandMakeVariables("includes", importsAttr);
+ if (importsAttr.startsWith("/")) {
+ ruleContext.attributeWarning("imports",
+ "ignoring invalid absolute path '" + importsAttr + "'");
+ continue;
+ }
+ PathFragment importsPath = packageFragment.getRelative(importsAttr).normalize();
+ if (!importsPath.isNormalized()) {
+ ruleContext.attributeError("imports",
+ "Path references a path above the execution root.");
+ }
+ result.add(importsPath);
+ }
+ return result;
+ }
+
+ @Override
public void createExecutable(RuleContext ruleContext, PyCommon common,
- CcLinkParamsStore ccLinkParamsStore) {
+ CcLinkParamsStore ccLinkParamsStore, NestedSet<PathFragment> imports) {
String main = common.determineMainExecutableSource();
BazelPythonConfiguration config = ruleContext.getFragment(BazelPythonConfiguration.class);
String pythonBinary;
@@ -82,7 +109,8 @@
STUB_TEMPLATE,
ImmutableList.of(
Substitution.of("%main%", main),
- Substitution.of("%python_binary%", pythonBinary)),
+ Substitution.of("%python_binary%", pythonBinary),
+ Substitution.of("%imports%", Joiner.on(":").join(imports))),
true));
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt
index fdc63cb..2e7677e 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt
@@ -33,6 +33,10 @@
# Case 4: Path has to be looked up in the search path.
return SearchPath(PYTHON_BINARY)
+def CreatePythonPathEntries(python_imports, module_space):
+ parts = python_imports.split(':');
+ return [module_space] + ["%s/%s" % (module_space, path) for path in parts]
+
def Main():
args = sys.argv[1:]
@@ -61,7 +65,8 @@
raise AssertionError('Cannot find .runfiles directory for %s' %
sys.argv[0])
- python_path_entries = [module_space]
+ python_imports = '%imports%'
+ python_path_entries = CreatePythonPathEntries(python_imports, module_space)
external_dir = os.path.join(module_space, 'external')
if os.path.isdir(external_dir):
diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyBinary.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyBinary.java
index 01829b6..ae048a0 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/python/PyBinary.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyBinary.java
@@ -21,10 +21,12 @@
import com.google.devtools.build.lib.analysis.Runfiles;
import com.google.devtools.build.lib.analysis.RunfilesProvider;
import com.google.devtools.build.lib.analysis.RunfilesSupport;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory;
import com.google.devtools.build.lib.rules.cpp.CcLinkParams;
import com.google.devtools.build.lib.rules.cpp.CcLinkParamsProvider;
import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore;
+import com.google.devtools.build.lib.vfs.PathFragment;
import java.util.ArrayList;
import java.util.List;
@@ -65,7 +67,12 @@
return null;
}
- semantics.createExecutable(ruleContext, common, ccLinkParamsStore);
+ NestedSet<PathFragment> imports = common.collectImports(ruleContext, semantics);
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
+ semantics.createExecutable(ruleContext, common, ccLinkParamsStore, imports);
Runfiles commonRunfiles = collectCommonRunfiles(ruleContext, common, semantics);
Runfiles.Builder defaultRunfilesBuilder = new Runfiles.Builder(ruleContext.getWorkspaceName())
@@ -99,7 +106,8 @@
.setFilesToBuild(common.getFilesToBuild())
.add(RunfilesProvider.class, runfilesProvider)
.setRunfilesSupport(runfilesSupport, common.getExecutable())
- .add(CcLinkParamsProvider.class, new CcLinkParamsProvider(ccLinkParamsStore));
+ .add(CcLinkParamsProvider.class, new CcLinkParamsProvider(ccLinkParamsStore))
+ .add(PythonImportsProvider.class, new PythonImportsProvider(imports));
}
private static Runfiles collectCommonRunfiles(RuleContext ruleContext, PyCommon common,
diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java
index 969955f..9b6813f 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java
@@ -276,14 +276,31 @@
}
private NestedSet<Artifact> collectTransitivePythonSources() {
- NestedSetBuilder<Artifact> builder =
- NestedSetBuilder.compileOrder();
+ NestedSetBuilder<Artifact> builder = NestedSetBuilder.compileOrder();
collectTransitivePythonSourcesFrom(getTargetDeps(), builder);
- addSourceFiles(builder, ruleContext
- .getPrerequisiteArtifacts("srcs", Mode.TARGET).filter(PyRuleClasses.PYTHON_SOURCE).list());
+ addSourceFiles(builder,
+ ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET)
+ .filter(PyRuleClasses.PYTHON_SOURCE).list());
return builder.build();
}
+ public NestedSet<PathFragment> collectImports(
+ RuleContext ruleContext, PythonSemantics semantics) {
+ NestedSetBuilder<PathFragment> builder = NestedSetBuilder.compileOrder();
+ builder.addAll(semantics.getImports(ruleContext));
+ collectTransitivePythonImports(builder);
+ return builder.build();
+ }
+
+ private void collectTransitivePythonImports(NestedSetBuilder<PathFragment> builder) {
+ for (TransitiveInfoCollection dep : getTargetDeps()) {
+ if (dep.getProvider(PythonImportsProvider.class) != null) {
+ PythonImportsProvider provider = dep.getProvider(PythonImportsProvider.class);
+ builder.addTransitive(provider.getTransitivePythonImports());
+ }
+ }
+ }
+
/**
* Checks that the source file version is compatible with the Python interpreter.
*/
diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyLibrary.java
index eb2e931..95f4c83 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/python/PyLibrary.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyLibrary.java
@@ -27,6 +27,7 @@
import com.google.devtools.build.lib.rules.cpp.CcLinkParams;
import com.google.devtools.build.lib.rules.cpp.CcLinkParamsProvider;
import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore;
+import com.google.devtools.build.lib.vfs.PathFragment;
import java.util.ArrayList;
import java.util.List;
@@ -50,7 +51,6 @@
common.validatePackageName();
semantics.validate(ruleContext, common);
-
List<Artifact> srcs = common.validateSrcs();
List<Artifact> allOutputs =
new ArrayList<>(semantics.precompiledPythonFiles(ruleContext, srcs, common));
@@ -69,6 +69,11 @@
}
};
+ NestedSet<PathFragment> imports = common.collectImports(ruleContext, semantics);
+ if (ruleContext.hasErrors()) {
+ return null;
+ }
+
Runfiles.Builder runfilesBuilder = new Runfiles.Builder(ruleContext.getWorkspaceName());
if (common.getConvertedFiles() != null) {
runfilesBuilder.addSymlinks(common.getConvertedFiles());
@@ -85,6 +90,7 @@
.setFilesToBuild(filesToBuild)
.add(RunfilesProvider.class, RunfilesProvider.simple(runfilesBuilder.build()))
.add(CcLinkParamsProvider.class, new CcLinkParamsProvider(ccLinkParamsStore))
+ .add(PythonImportsProvider.class, new PythonImportsProvider(imports))
.build();
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonImportsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonImportsProvider.java
new file mode 100644
index 0000000..03bb52a
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonImportsProvider.java
@@ -0,0 +1,36 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.rules.python;
+
+import com.google.devtools.build.lib.analysis.TransitiveInfoProvider;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
+import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * A {@link TransitiveInfoProvider} that supplies import directories for Python dependencies.
+ */
+@Immutable
+public final class PythonImportsProvider implements TransitiveInfoProvider {
+
+ private final NestedSet<PathFragment> transitivePythonImports;
+
+ public PythonImportsProvider(NestedSet<PathFragment> transitivePythonImports) {
+ this.transitivePythonImports = transitivePythonImports;
+ }
+
+ public NestedSet<PathFragment> getTransitivePythonImports() {
+ return transitivePythonImports;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java
index abf6a88..2168a87 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/python/PythonSemantics.java
@@ -17,10 +17,13 @@
import com.google.devtools.build.lib.analysis.RuleContext;
import com.google.devtools.build.lib.analysis.Runfiles;
import com.google.devtools.build.lib.analysis.RunfilesSupport;
+import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore;
import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.InstrumentationSpec;
+import com.google.devtools.build.lib.vfs.PathFragment;
import java.util.Collection;
+import java.util.List;
/**
* Pluggable semantics for Python rules.
@@ -56,16 +59,21 @@
RuleContext ruleContext, Collection<Artifact> sources, PyCommon common);
/**
+ * Returns a list of PathFragments for the import paths specified in the imports attribute.
+ */
+ List<PathFragment> getImports(RuleContext ruleContext);
+
+ /**
* Create the actual executable artifact.
*
* <p>This should create a generating action for {@code common.getExecutable()}.
*/
void createExecutable(RuleContext ruleContext, PyCommon common,
- CcLinkParamsStore ccLinkParamsStore);
+ CcLinkParamsStore ccLinkParamsStore, NestedSet<PathFragment> imports);
/**
* Called at the end of the analysis of {@code py_binary} rules.
- * @throws InterruptedException
+ * @throws InterruptedException
*/
void postInitBinary(RuleContext ruleContext, RunfilesSupport runfilesSupport,
PyCommon common) throws InterruptedException;
