python,runfiles: move to different package
Move the Python runfiles library from
`@bazel_tools//tools/runfiles:py-runfiles` to
`@bazel_tools//tools/python/runfiles:runfiles`
Also rename the testdata runfiles.py to foo.py.
This file was not a mock runfiles library, just a
client file using the runfiles library that was
also called runfiles.py
Fixes https://github.com/bazelbuild/bazel/issues/4878
Change-Id: I874b230c93679d4454ac91e816932c8272ecc5c7
Closes #4981.
Change-Id: I908e0ab7ec61225e82f70793b1a05432e7f0b07e
PiperOrigin-RevId: 192256481
diff --git a/tools/python/BUILD b/tools/python/BUILD
index 6933f9d..ab09128 100644
--- a/tools/python/BUILD
+++ b/tools/python/BUILD
@@ -6,10 +6,28 @@
)
filegroup(
- name = "srcs",
+ name = "srcs_and_embedded_tools",
srcs = [
# Tools are build from the workspace for tests.
"2to3.sh",
"BUILD",
],
+ visibility = ["//visibility:private"],
+)
+
+filegroup(
+ name = "srcs",
+ srcs = [
+ ":srcs_and_embedded_tools",
+ "//tools/python/runfiles:srcs",
+ ],
+)
+
+filegroup(
+ name = "embedded_tools",
+ srcs = [
+ ":srcs_and_embedded_tools",
+ "//tools/python/runfiles:embedded_tools",
+ ],
+ visibility = ["//tools:__pkg__"],
)
diff --git a/tools/python/runfiles/BUILD b/tools/python/runfiles/BUILD
new file mode 100644
index 0000000..79273e8
--- /dev/null
+++ b/tools/python/runfiles/BUILD
@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:private"])
+
+filegroup(
+ name = "srcs",
+ srcs = glob(
+ ["**"],
+ exclude = [
+ ".*",
+ "*~",
+ ],
+ ),
+ visibility = ["//tools/python:__pkg__"],
+)
+
+filegroup(
+ name = "embedded_tools",
+ srcs = [
+ "BUILD.tools",
+ "runfiles.py",
+ ],
+ visibility = ["//tools/python:__pkg__"],
+)
+
+py_library(
+ name = "runfiles",
+ testonly = 1,
+ srcs = ["runfiles.py"],
+)
+
+py_test(
+ name = "runfiles_test",
+ srcs = ["runfiles_test.py"],
+ visibility = ["//visibility:public"],
+ deps = [":runfiles"],
+)
diff --git a/tools/python/runfiles/BUILD.tools b/tools/python/runfiles/BUILD.tools
new file mode 100644
index 0000000..4080d68
--- /dev/null
+++ b/tools/python/runfiles/BUILD.tools
@@ -0,0 +1,5 @@
+py_library(
+ name = "runfiles",
+ srcs = ["runfiles.py"],
+ visibility = ["//visibility:public"],
+)
diff --git a/tools/python/runfiles/runfiles.py b/tools/python/runfiles/runfiles.py
new file mode 100644
index 0000000..17e4121
--- /dev/null
+++ b/tools/python/runfiles/runfiles.py
@@ -0,0 +1,221 @@
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Runfiles lookup library for Bazel-built Python binaries and tests.
+
+Usage:
+
+from bazel_tools.tools.python.runfiles import runfiles
+
+r = runfiles.Create()
+with open(r.Rlocation("io_bazel/foo/bar.txt"), "r") as f:
+ contents = f.readlines()
+
+The code above creates a manifest- or directory-based implementations based on
+the environment variables in os.environ. See `Create()` for more info.
+
+If you want to explicitly create a manifest- or directory-based
+implementations, you can do so as follows:
+
+ r1 = runfiles.CreateManifestBased("path/to/foo.runfiles_manifest")
+
+ r2 = runfiles.CreateDirectoryBased("path/to/foo.runfiles/")
+
+If you want to start subprocesses that also need runfiles, you need to set the
+right environment variables for them:
+
+ import subprocess
+ from bazel_tools.tools.python.runfiles import runfiles
+
+ r = runfiles.Create()
+ env = {}
+ ...
+ env.update(r.EnvVars())
+ p = subprocess.Popen([r.Rlocation("path/to/binary")], env, ...)
+"""
+
+import os
+import posixpath
+
+
+def CreateManifestBased(manifest_path):
+ return _Runfiles(_ManifestBased(manifest_path))
+
+
+def CreateDirectoryBased(runfiles_dir_path):
+ return _Runfiles(_DirectoryBased(runfiles_dir_path))
+
+
+def Create(env=None):
+ """Returns a new `Runfiles` instance.
+
+ The returned object is either:
+ - manifest-based, meaning it looks up runfile paths from a manifest file, or
+ - directory-based, meaning it looks up runfile paths under a given directory
+ path
+
+ If `env` contains "RUNFILES_MANIFEST_FILE" with non-empty value, this method
+ returns a manifest-based implementation. The object eagerly reads and caches
+ the whole manifest file upon instantiation; this may be relevant for
+ performance consideration.
+
+ Otherwise, if `env` contains "RUNFILES_DIR" with non-empty value (checked in
+ this priority order), this method returns a directory-based implementation.
+
+ If neither cases apply, this method returns null.
+
+ Args:
+ env: {string: string}; optional; the map of environment variables. If None,
+ this function uses the environment variable map of this process.
+ Raises:
+ IOError: if some IO error occurs.
+ """
+ env_map = os.environ if env is None else env
+ manifest = env_map.get("RUNFILES_MANIFEST_FILE")
+ if manifest:
+ return CreateManifestBased(manifest)
+
+ directory = env_map.get("RUNFILES_DIR")
+ if directory:
+ return CreateDirectoryBased(directory)
+
+ return None
+
+
+class _Runfiles(object):
+ """Returns the runtime location of runfiles.
+
+ Runfiles are data-dependencies of Bazel-built binaries and tests.
+ """
+
+ def __init__(self, strategy):
+ self._strategy = strategy
+
+ def Rlocation(self, path):
+ """Returns the runtime path of a runfile.
+
+ Runfiles are data-dependencies of Bazel-built binaries and tests.
+
+ The returned path may not be valid. The caller should check the path's
+ validity and that the path exists.
+
+ The function may return None. In that case the caller can be sure that the
+ rule does not know about this data-dependency.
+
+ Args:
+ path: string; runfiles-root-relative path of the runfile
+ Returns:
+ the path to the runfile, which the caller should check for existence, or
+ None if the method doesn't know about this runfile
+ Raises:
+ TypeError: if `path` is not a string
+ ValueError: if `path` is None or empty, or it's absolute or contains
+ uplevel references
+ """
+ if not path:
+ raise ValueError()
+ if not isinstance(path, str):
+ raise TypeError()
+ if ".." in path:
+ raise ValueError("path contains uplevel references: \"%s\"" % path)
+ if path[0] == "\\":
+ raise ValueError("path is absolute without a drive letter: \"%s\"" % path)
+ if os.path.isabs(path):
+ return path
+ return self._strategy.RlocationChecked(path)
+
+ def EnvVars(self):
+ """Returns environment variables for subprocesses.
+
+ The caller should set the returned key-value pairs in the environment of
+ subprocesses in case those subprocesses are also Bazel-built binaries that
+ need to use runfiles.
+
+ Returns:
+ {string: string}; a dict; keys are environment variable names, values are
+ the values for these environment variables
+ """
+ return self._strategy.EnvVars()
+
+
+class _ManifestBased(object):
+ """`Runfiles` strategy that parses a runfiles-manifest to look up runfiles."""
+
+ def __init__(self, path):
+ if not path:
+ raise ValueError()
+ if not isinstance(path, str):
+ raise TypeError()
+ self._path = path
+ self._runfiles = _ManifestBased._LoadRunfiles(path)
+
+ def RlocationChecked(self, path):
+ return self._runfiles.get(path)
+
+ @staticmethod
+ def _LoadRunfiles(path):
+ """Loads the runfiles manifest."""
+ result = {}
+ with open(path, "r") as f:
+ for line in f:
+ line = line.strip()
+ if line:
+ tokens = line.split(" ", 1)
+ if len(tokens) == 1:
+ result[line] = line
+ else:
+ result[tokens[0]] = tokens[1]
+ return result
+
+ def _GetRunfilesDir(self):
+ if self._path.endswith("/MANIFEST") or self._path.endswith("\\MANIFEST"):
+ return self._path[:-len("/MANIFEST")]
+ elif self._path.endswith(".runfiles_manifest"):
+ return self._path[:-len("_manifest")]
+ else:
+ return ""
+
+ def EnvVars(self):
+ directory = self._GetRunfilesDir()
+ return {
+ "RUNFILES_MANIFEST_FILE": self._path,
+ "RUNFILES_DIR": directory,
+ # TODO(laszlocsomor): remove JAVA_RUNFILES once the Java launcher can
+ # pick up RUNFILES_DIR.
+ "JAVA_RUNFILES": directory,
+ }
+
+
+class _DirectoryBased(object):
+ """`Runfiles` strategy that appends runfiles paths to the runfiles root."""
+
+ def __init__(self, path):
+ if not path:
+ raise ValueError()
+ if not isinstance(path, str):
+ raise TypeError()
+ self._runfiles_root = path
+
+ def RlocationChecked(self, path):
+ # Use posixpath instead of os.path, because Bazel only creates a runfiles
+ # tree on Unix platforms, so `Create()` will only create a directory-based
+ # runfiles strategy on those platforms.
+ return posixpath.join(self._runfiles_root, path)
+
+ def EnvVars(self):
+ return {
+ "RUNFILES_DIR": self._runfiles_root,
+ # TODO(laszlocsomor): remove JAVA_RUNFILES once the Java launcher can
+ # pick up RUNFILES_DIR.
+ "JAVA_RUNFILES": self._runfiles_root,
+ }
diff --git a/tools/python/runfiles/runfiles_test.py b/tools/python/runfiles/runfiles_test.py
new file mode 100644
index 0000000..bbe1c84
--- /dev/null
+++ b/tools/python/runfiles/runfiles_test.py
@@ -0,0 +1,182 @@
+# pylint: disable=g-bad-file-header
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import unittest
+
+from tools.python.runfiles import runfiles
+
+
+class RunfilesTest(unittest.TestCase):
+ # """Unit tests for `runfiles.Runfiles`."""
+
+ def testRlocationArgumentValidation(self):
+ r = runfiles.Create({"RUNFILES_DIR": "whatever"})
+ self.assertRaises(ValueError, lambda: r.Rlocation(None))
+ self.assertRaises(ValueError, lambda: r.Rlocation(""))
+ self.assertRaises(TypeError, lambda: r.Rlocation(1))
+ self.assertRaisesRegexp(ValueError, "contains uplevel",
+ lambda: r.Rlocation("foo/.."))
+ self.assertRaisesRegexp(ValueError, "is absolute without a drive letter",
+ lambda: r.Rlocation("\\foo"))
+
+ def testCreatesManifestBasedRunfiles(self):
+ with _MockFile(contents=["a/b c/d"]) as mf:
+ r = runfiles.Create({
+ "RUNFILES_MANIFEST_FILE": mf.Path(),
+ "RUNFILES_DIR": "ignored when RUNFILES_MANIFEST_FILE has a value",
+ "TEST_SRCDIR": "always ignored",
+ })
+ self.assertEqual(r.Rlocation("a/b"), "c/d")
+ self.assertIsNone(r.Rlocation("foo"))
+
+ def testManifestBasedRunfilesEnvVars(self):
+ with _MockFile(name="MANIFEST") as mf:
+ r = runfiles.Create({
+ "RUNFILES_MANIFEST_FILE": mf.Path(),
+ "TEST_SRCDIR": "always ignored",
+ })
+ self.assertDictEqual(
+ r.EnvVars(), {
+ "RUNFILES_MANIFEST_FILE": mf.Path(),
+ "RUNFILES_DIR": mf.Path()[:-len("/MANIFEST")],
+ "JAVA_RUNFILES": mf.Path()[:-len("/MANIFEST")],
+ })
+
+ with _MockFile(name="foo.runfiles_manifest") as mf:
+ r = runfiles.Create({
+ "RUNFILES_MANIFEST_FILE": mf.Path(),
+ "TEST_SRCDIR": "always ignored",
+ })
+ self.assertDictEqual(
+ r.EnvVars(), {
+ "RUNFILES_MANIFEST_FILE":
+ mf.Path(),
+ "RUNFILES_DIR": (
+ mf.Path()[:-len("foo.runfiles_manifest")] + "foo.runfiles"),
+ "JAVA_RUNFILES": (
+ mf.Path()[:-len("foo.runfiles_manifest")] + "foo.runfiles"),
+ })
+
+ with _MockFile(name="x_manifest") as mf:
+ r = runfiles.Create({
+ "RUNFILES_MANIFEST_FILE": mf.Path(),
+ "TEST_SRCDIR": "always ignored",
+ })
+ self.assertDictEqual(
+ r.EnvVars(), {
+ "RUNFILES_MANIFEST_FILE": mf.Path(),
+ "RUNFILES_DIR": "",
+ "JAVA_RUNFILES": "",
+ })
+
+ def testCreatesDirectoryBasedRunfiles(self):
+ r = runfiles.Create({
+ "RUNFILES_DIR": "runfiles/dir",
+ "TEST_SRCDIR": "always ignored",
+ })
+ self.assertEqual(r.Rlocation("a/b"), "runfiles/dir/a/b")
+ self.assertEqual(r.Rlocation("foo"), "runfiles/dir/foo")
+
+ def testDirectoryBasedRunfilesEnvVars(self):
+ r = runfiles.Create({
+ "RUNFILES_DIR": "runfiles/dir",
+ "TEST_SRCDIR": "always ignored",
+ })
+ self.assertDictEqual(r.EnvVars(), {
+ "RUNFILES_DIR": "runfiles/dir",
+ "JAVA_RUNFILES": "runfiles/dir",
+ })
+
+ def testFailsToCreateManifestBasedBecauseManifestDoesNotExist(self):
+
+ def _Run():
+ runfiles.Create({"RUNFILES_MANIFEST_FILE": "non-existing path"})
+
+ self.assertRaisesRegexp(IOError, "non-existing path", _Run)
+
+ def testFailsToCreateAnyRunfilesBecauseEnvvarsAreNotDefined(self):
+ with _MockFile(contents=["a b"]) as mf:
+ runfiles.Create({
+ "RUNFILES_MANIFEST_FILE": mf.Path(),
+ "RUNFILES_DIR": "whatever",
+ "TEST_SRCDIR": "always ignored",
+ })
+ runfiles.Create({
+ "RUNFILES_DIR": "whatever",
+ "TEST_SRCDIR": "always ignored",
+ })
+ self.assertIsNone(runfiles.Create({"TEST_SRCDIR": "always ignored"}))
+ self.assertIsNone(runfiles.Create({"FOO": "bar"}))
+
+ def testManifestBasedRlocation(self):
+ with _MockFile(contents=[
+ "Foo/runfile1", "Foo/runfile2 C:/Actual Path\\runfile2",
+ "Foo/Bar/runfile3 D:\\the path\\run file 3.txt"
+ ]) as mf:
+ r = runfiles.CreateManifestBased(mf.Path())
+ self.assertEqual(r.Rlocation("Foo/runfile1"), "Foo/runfile1")
+ self.assertEqual(r.Rlocation("Foo/runfile2"), "C:/Actual Path\\runfile2")
+ self.assertEqual(
+ r.Rlocation("Foo/Bar/runfile3"), "D:\\the path\\run file 3.txt")
+ self.assertIsNone(r.Rlocation("unknown"))
+ if RunfilesTest.IsWindows():
+ self.assertEqual(r.Rlocation("c:/foo"), "c:/foo")
+ self.assertEqual(r.Rlocation("c:\\foo"), "c:\\foo")
+ else:
+ self.assertEqual(r.Rlocation("/foo"), "/foo")
+
+ def testDirectoryBasedRlocation(self):
+ # The _DirectoryBased strategy simply joins the runfiles directory and the
+ # runfile's path on a "/". This strategy does not perform any normalization,
+ # nor does it check that the path exists.
+ r = runfiles.CreateDirectoryBased("foo/bar baz//qux/")
+ self.assertEqual(r.Rlocation("arg"), "foo/bar baz//qux/arg")
+ if RunfilesTest.IsWindows():
+ self.assertEqual(r.Rlocation("c:/foo"), "c:/foo")
+ self.assertEqual(r.Rlocation("c:\\foo"), "c:\\foo")
+ else:
+ self.assertEqual(r.Rlocation("/foo"), "/foo")
+
+ @staticmethod
+ def IsWindows():
+ return os.name == "nt"
+
+
+class _MockFile(object):
+
+ def __init__(self, name=None, contents=None):
+ self._contents = contents or []
+ self._name = name or "x"
+ self._path = None
+
+ def __enter__(self):
+ tmpdir = os.environ.get("TEST_TMPDIR")
+ self._path = os.path.join(tempfile.mkdtemp(dir=tmpdir), self._name)
+ with open(self._path, "wt") as f:
+ f.writelines(l + "\n" for l in self._contents)
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ os.remove(self._path)
+ os.rmdir(os.path.dirname(self._path))
+
+ def Path(self):
+ return self._path
+
+
+if __name__ == "__main__":
+ unittest.main()
