Using stub_template.txt as __main__.py and zip header in python executable zip
In stub_template.txt, we now unzip the python zip file to a temp
directory if needed.
This will get rid of bash in python executable file completely.
Users can run the binary directly or using python <zip>, like:
./bazel-bin/examples/py_native/bin
or python ./bazel-bin/examples/py_native/bin
On Windows, we can use the second way to run python binary from native
Windows command line (cmd.exe).
--
Change-Id: I73fdd88f05f8f343dd19b2f3686ae031dfb476ba
Reviewed-on: https://bazel-review.googlesource.com/#/c/5310
MOS_MIGRATED_REVID=129767890
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java
index 3ea19fe..3d95040 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java
@@ -41,7 +41,6 @@
import com.google.devtools.build.lib.syntax.Type;
import com.google.devtools.build.lib.util.FileTypeSet;
import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -105,6 +104,16 @@
return result;
}
+ /** @return An artifact next to the executable file with ".zip" suffix */
+ public Artifact getPythonZipArtifact(RuleContext ruleContext, Artifact executable) {
+ return ruleContext.getRelatedArtifact(executable.getRootRelativePath(), ".zip");
+ }
+
+ /** @return An artifact next to the executable file with ".temp" suffix */
+ public Artifact getPythonTemplateMainArtifact(RuleContext ruleContext, Artifact executable) {
+ return ruleContext.getRelatedArtifact(executable.getRootRelativePath(), ".temp");
+ }
+
@Override
public void createExecutable(
RuleContext ruleContext,
@@ -133,46 +142,45 @@
Substitution.of("%main%", main),
Substitution.of("%python_binary%", pythonBinary),
Substitution.of("%imports%", Joiner.on(":").join(imports)),
- Substitution.of("%workspace_name%", ruleContext.getWorkspaceName())),
+ Substitution.of("%workspace_name%", ruleContext.getWorkspaceName()),
+ Substitution.of("%is_zipfile%", "False")),
true));
} else {
- Artifact zipFile = common.getPythonZipArtifact();
+ Artifact zipFile = getPythonZipArtifact(ruleContext, executable);
+ Artifact templateMain = getPythonTemplateMainArtifact(ruleContext, executable);
PathFragment workspaceName = getWorkspaceNameForPythonZip(ruleContext.getWorkspaceName());
main = workspaceName.getRelative(common.determineMainExecutableSource(false)).toString();
PathFragment defaultWorkspacename = new PathFragment(Label.DEFAULT_REPOSITORY_DIRECTORY);
- StringBuilder importPaths = new StringBuilder();
- importPaths.append(File.pathSeparator).append("$PYTHON_RUNFILES/").append(workspaceName);
+ List<PathFragment> importPaths = new ArrayList<>();
for (PathFragment path : imports) {
if (path.startsWith(defaultWorkspacename)) {
path = new PathFragment(workspaceName, path.subFragment(1, path.segmentCount()));
}
- importPaths.append(File.pathSeparator).append("$PYTHON_RUNFILES/").append(path);
+ importPaths.add(path);
}
// The executable zip file wil unzip itself into a tmp directory and then run from there
- String zipHeader =
- "#!/bin/sh\n"
- + "export TMPDIR=${TMPDIR:-/tmp/Bazel}\n"
- + "mkdir -p \"${TMPDIR}\"\n"
- + "export PYTHON_RUNFILES=$(mktemp -d \"${TMPDIR%%/}/runfiles.XXXXXXXX\")\n"
- + "export PYTHONPATH=\"$PYTHONPATH"
- + importPaths
- + "\"\n"
- + "unzip -q -o $0 -d \"$PYTHON_RUNFILES\" 2> /dev/null\n"
- + "retCode=0\n"
- + pythonBinary
- + " \"$PYTHON_RUNFILES/"
- + main
- + "\" $@ || retCode=$?\n"
- + "rm -rf \"$PYTHON_RUNFILES\"\n"
- + "exit $retCode\n";
+ ruleContext.registerAction(
+ new TemplateExpansionAction(
+ ruleContext.getActionOwner(),
+ templateMain,
+ STUB_TEMPLATE,
+ ImmutableList.of(
+ Substitution.of("%main%", main),
+ Substitution.of("%python_binary%", pythonBinary),
+ Substitution.of("%imports%", Joiner.on(":").join(importPaths)),
+ Substitution.of("%workspace_name%", ruleContext.getWorkspaceName()),
+ Substitution.of("%is_zipfile%", "True")),
+ true));
+
ruleContext.registerAction(
new SpawnAction.Builder()
.addInput(zipFile)
+ .addInput(templateMain)
.addOutput(executable)
.setShellCommand(
- "echo '"
- + zipHeader
- + "' | cat - "
+ "cat "
+ + templateMain.getExecPathString()
+ + " "
+ zipFile.getExecPathString()
+ " > "
+ executable.getExecPathString())
@@ -187,12 +195,13 @@
PyCommon common) throws InterruptedException {
if (ruleContext.getConfiguration().buildPythonZip()) {
FilesToRunProvider zipper = ruleContext.getExecutablePrerequisite("$zipper", Mode.HOST);
+ Artifact executable = common.getExecutable();
if (!ruleContext.hasErrors()) {
createPythonZipAction(
ruleContext,
- common.getExecutable(),
- common.getPythonZipArtifact(),
- common.determineMainExecutableSource(false),
+ executable,
+ getPythonZipArtifact(ruleContext, executable),
+ getPythonTemplateMainArtifact(ruleContext, executable),
zipper,
runfilesSupport);
}
@@ -230,15 +239,15 @@
RuleContext ruleContext,
Artifact executable,
Artifact zipFile,
- String main,
+ Artifact templateMain,
FilesToRunProvider zipper,
RunfilesSupport runfilesSupport) {
NestedSetBuilder<Artifact> inputsBuilder = NestedSetBuilder.stableOrder();
PathFragment workspaceName = getWorkspaceNameForPythonZip(ruleContext.getWorkspaceName());
CustomCommandLine.Builder argv = new CustomCommandLine.Builder();
-
- argv.add("__main__.py=" + main);
+ inputsBuilder.add(templateMain);
+ argv.add("__main__.py=" + templateMain.getExecPathString());
// Creating __init__.py files under each directory
argv.add("__init__.py=");
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt
index 8e561a5..ac0b8bd 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt
@@ -1,9 +1,12 @@
-#!/usr/bin/python
+#!/usr/bin/env python
import os
import re
+import tempfile
+import shutil
import sys
import subprocess
+import zipfile
# Return True if running on Windows
def IsWindows():
@@ -23,6 +26,9 @@
return path
return None
+def IsRunningFromZip():
+ return %is_zipfile%
+
# Find the real Python binary if it's not a normal absolute path
def FindPythonBinary():
if PYTHON_BINARY.startswith('//'):
@@ -43,11 +49,8 @@
parts = python_imports.split(':');
return [module_space] + ["%s/%s" % (module_space, path) for path in parts]
-def Main():
- args = sys.argv[1:]
-
- new_env = {}
-
+# Find the runfiles tree
+def FindModuleSpace():
# Follow symlinks, looking for my module space
stub_filename = os.path.abspath(sys.argv[0])
while True:
@@ -73,6 +76,24 @@
raise AssertionError('Cannot find .runfiles directory for %s' %
sys.argv[0])
+ return module_space
+
+# Create the runfiles tree by extracting the zip file
+def CreateModuleSpace():
+ module_space = tempfile.mkdtemp("", "Bazel.runfiles_")
+ zf = zipfile.ZipFile(os.path.dirname(__file__))
+ zf.extractall(module_space)
+ return module_space
+
+def Main():
+ args = sys.argv[1:]
+
+ new_env = {}
+
+ if IsRunningFromZip():
+ module_space = CreateModuleSpace()
+ else:
+ module_space = FindModuleSpace()
python_imports = '%imports%'
python_path_entries = CreatePythonPathEntries(python_imports, module_space)
@@ -109,7 +130,12 @@
try:
sys.stdout.flush()
- os.execv(args[0], args)
+ if IsRunningFromZip():
+ retCode = subprocess.call(args)
+ shutil.rmtree(module_space, True)
+ exit(retCode)
+ else:
+ os.execv(args[0], args)
except EnvironmentError as e:
# This exception occurs when os.execv() fails for some reason.
if not getattr(e, 'filename', None):
diff --git a/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java
index d03f380..ab3f321 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/python/PyCommon.java
@@ -448,12 +448,6 @@
public Artifact getExecutable() {
return executable;
}
- /** @return An artifact next to the executable file with ".zip" suffix */
- public Artifact getPythonZipArtifact() {
- PathFragment original = executable.getRootRelativePath();
- return ruleContext.getDerivedArtifact(
- original.replaceName(original.getBaseName() + ".zip"), executable.getRoot());
- }
public Map<PathFragment, Artifact> getConvertedFiles() {
return convertedFiles;
diff --git a/src/test/shell/bazel/bazel_example_test.sh b/src/test/shell/bazel/bazel_example_test.sh
index 93075fd..c440a6e 100755
--- a/src/test/shell/bazel/bazel_example_test.sh
+++ b/src/test/shell/bazel/bazel_example_test.sh
@@ -113,6 +113,20 @@
assert_test_fails //examples/py_native:fail --python2_path=python
}
+function test_native_python_with_zip() {
+ assert_build //examples/py_native:bin --python2_path=python --build_python_zip
+ # run the python package directly
+ ./bazel-bin/examples/py_native/bin >& $TEST_log \
+ || fail "//examples/py_native:bin execution failed"
+ expect_log "Fib(5) == 8"
+ # Using python <zipfile> to run the python package
+ python ./bazel-bin/examples/py_native/bin >& $TEST_log \
+ || fail "//examples/py_native:bin execution failed"
+ expect_log "Fib(5) == 8"
+ assert_test_ok //examples/py_native:test --python2_path=python --build_python_zip
+ assert_test_fails //examples/py_native:fail --python2_path=python --build_python_zip
+}
+
function test_shell() {
assert_build "//examples/shell:bin"
assert_bazel_run "//examples/shell:bin" "Hello Bazel!"
diff --git a/src/test/shell/bazel/bazel_windows_example_test.sh b/src/test/shell/bazel/bazel_windows_example_test.sh
index c0bd25b..ab31786 100755
--- a/src/test/shell/bazel/bazel_windows_example_test.sh
+++ b/src/test/shell/bazel/bazel_windows_example_test.sh
@@ -84,5 +84,25 @@
assert_test_fails "${java_native_tests}:resource-fail"
}
+function test_native_python() {
+ assert_build //examples/py_native:bin --python2_path=python
+ assert_test_ok //examples/py_native:test --python2_path=python
+ assert_test_fails //examples/py_native:fail --python2_path=python
+}
+
+function test_native_python_with_zip() {
+ assert_build //examples/py_native:bin --python2_path=python --build_python_zip
+ # run the python package directly
+ ./bazel-bin/examples/py_native/bin >& $TEST_log \
+ || fail "//examples/py_native:bin execution failed"
+ expect_log "Fib(5) == 8"
+ # Using python <zipfile> to run the python package
+ python ./bazel-bin/examples/py_native/bin >& $TEST_log \
+ || fail "//examples/py_native:bin execution failed"
+ expect_log "Fib(5) == 8"
+ assert_test_ok //examples/py_native:test --python2_path=python --build_python_zip
+ assert_test_fails //examples/py_native:fail --python2_path=python --build_python_zip
+}
+
run_suite "examples on Windows"
