Android,Windows: support long paths in tooling
aar_resources_extractor now supports long paths
on Windows.
If the script needs to extract a file from the
AAR where the destination path is too long, the
script will:
1. create a temporary junction under a short path,
pointing to the destination directory (which
has a long path)
2. extract the file under the junction
3. delete the junction and the temp directory
See https://github.com/bazelbuild/bazel/issues/3659
Change-Id: Ie85665b360a6514afaac546aaec8869224fe9d06
PiperOrigin-RevId: 167545085
diff --git a/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java b/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java
index 61295fc..e3d5a12 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/android/AarImport.java
@@ -176,6 +176,7 @@
private static Action[] createSingleFileExtractorActions(RuleContext ruleContext, Artifact aar,
String filename, Artifact outputArtifact) {
return new SpawnAction.Builder()
+ .useDefaultShellEnvironment()
.setExecutable(ruleContext.getExecutablePrerequisite(AarImportBaseRule.ZIPPER, Mode.HOST))
.setMnemonic("AarFileExtractor")
.setProgressMessage("Extracting %s from %s", filename, aar.getFilename())
@@ -193,6 +194,7 @@
private static Action[] createAarResourcesExtractorActions(
RuleContext ruleContext, Artifact aar, Artifact outputTree) {
return new SpawnAction.Builder()
+ .useDefaultShellEnvironment()
.setExecutable(
ruleContext.getExecutablePrerequisite(
AarImportBaseRule.AAR_RESOURCES_EXTRACTOR, Mode.HOST))
@@ -210,6 +212,7 @@
private static Action[] createAarEmbeddedJarsExtractorActions(RuleContext ruleContext,
Artifact aar, Artifact jarsTreeArtifact, Artifact singleJarParamFile) {
return new SpawnAction.Builder()
+ .useDefaultShellEnvironment()
.setExecutable(
ruleContext.getExecutablePrerequisite(
AarImportBaseRule.AAR_EMBEDDED_JARS_EXTACTOR, Mode.HOST))
@@ -248,6 +251,7 @@
Artifact outputZip) {
SpawnAction.Builder actionBuilder =
new SpawnAction.Builder()
+ .useDefaultShellEnvironment()
.setExecutable(
ruleContext.getExecutablePrerequisite(
AarImportBaseRule.AAR_NATIVE_LIBS_ZIP_CREATOR, Mode.HOST))
@@ -276,7 +280,7 @@
// Adds the appropriate SpawnAction options depending on if SingleJar is a jar or not.
private static SpawnAction.Builder singleJarSpawnActionBuilder(RuleContext ruleContext) {
- SpawnAction.Builder builder = new SpawnAction.Builder();
+ SpawnAction.Builder builder = new SpawnAction.Builder().useDefaultShellEnvironment();
Artifact singleJar = JavaToolchainProvider.fromRuleContext(ruleContext).getSingleJar();
if (singleJar.getFilename().endsWith(".jar")) {
builder
diff --git a/src/test/py/bazel/BUILD b/src/test/py/bazel/BUILD
index 3be9bc2..f93093b 100644
--- a/src/test/py/bazel/BUILD
+++ b/src/test/py/bazel/BUILD
@@ -20,6 +20,7 @@
visibility = [
"//src/test/py/bazel:__pkg__",
"//third_party/def_parser:__pkg__",
+ "//tools/android:__pkg__",
],
)
diff --git a/tools/android/BUILD b/tools/android/BUILD
index 75a9956..340847e 100644
--- a/tools/android/BUILD
+++ b/tools/android/BUILD
@@ -135,7 +135,10 @@
py_binary(
name = "aar_resources_extractor",
srcs = ["aar_resources_extractor.py"],
- deps = ["//third_party/py/gflags"],
+ deps = [
+ ":junction_lib",
+ "//third_party/py/gflags",
+ ],
)
py_test(
@@ -155,6 +158,30 @@
deps = [":resource_extractor"],
)
+py_library(
+ name = "junction_lib",
+ srcs = ["junction.py"],
+ visibility = ["//visibility:private"],
+)
+
+py_test(
+ name = "junction_test",
+ srcs = select({
+ "//src:windows": ["junction_test.py"],
+ "//src:windows_msvc": ["junction_test.py"],
+ "//conditions:default": ["dummy_test.py"],
+ }),
+ main = select({
+ "//src:windows": "junction_test.py",
+ "//src:windows_msvc": "junction_test.py",
+ "//conditions:default": "dummy_test.py",
+ }),
+ deps = [
+ ":junction_lib",
+ "//src/test/py/bazel:test_base",
+ ],
+)
+
filegroup(
name = "srcs",
srcs = glob(["**"]) + ["//tools/android/emulator:srcs"],
diff --git a/tools/android/BUILD.tools b/tools/android/BUILD.tools
index fde4587..7110448 100644
--- a/tools/android/BUILD.tools
+++ b/tools/android/BUILD.tools
@@ -189,7 +189,10 @@
py_binary(
name = "aar_resources_extractor",
srcs = ["aar_resources_extractor.py"],
- deps = ["//third_party/py/gflags"],
+ deps = [
+ ":junction_lib",
+ "//third_party/py/gflags",
+ ],
)
py_binary(
@@ -197,6 +200,12 @@
srcs = ["resource_extractor.py"],
)
+py_library(
+ name = "junction_lib",
+ srcs = ["junction.py"],
+ visibility = ["//visibility:private"],
+)
+
alias(
name = "android_runtest",
actual = "fail.sh",
diff --git a/tools/android/aar_resources_extractor.py b/tools/android/aar_resources_extractor.py
index 0e259b8..eb0d432 100644
--- a/tools/android/aar_resources_extractor.py
+++ b/tools/android/aar_resources_extractor.py
@@ -26,6 +26,7 @@
import sys
import zipfile
+from tools.android import junction
from third_party.py import gflags
FLAGS = gflags.FLAGS
@@ -37,10 +38,19 @@
def ExtractResources(aar, output_res_dir):
+ """Extract resource from an `aar` file to the `output_res_dir` directory."""
aar_contains_no_resources = True
+ output_res_dir_abs = os.path.normpath(
+ os.path.join(os.getcwd(), output_res_dir))
for name in aar.namelist():
if name.startswith("res/"):
- aar.extract(name, output_res_dir)
+ fullpath = os.path.normpath(os.path.join(output_res_dir_abs, name))
+ if os.name == "nt" and len(fullpath) >= 260: # MAX_PATH in <windows.h>
+ with junction.TempJunction(os.path.dirname(fullpath)) as juncpath:
+ shortpath = os.path.join(juncpath, os.path.basename(fullpath))
+ aar.extract(name, shortpath)
+ else:
+ aar.extract(name, output_res_dir)
aar_contains_no_resources = False
if aar_contains_no_resources:
empty_xml_filename = output_res_dir + "/res/values/empty.xml"
diff --git a/tools/android/dummy_test.py b/tools/android/dummy_test.py
new file mode 100644
index 0000000..9afac3f
--- /dev/null
+++ b/tools/android/dummy_test.py
@@ -0,0 +1,22 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Dummy empty test.
+
+You can use this as a dummy test for platform-specific tests on platforms that
+are not under test. For example, we use this dummy test on Linux in a
+Windows-specific py_test rule.
+"""
+
+if __name__ == "__main__":
+ pass
diff --git a/tools/android/junction.py b/tools/android/junction.py
new file mode 100644
index 0000000..7dc58b8
--- /dev/null
+++ b/tools/android/junction.py
@@ -0,0 +1,115 @@
+# pylint: disable=g-direct-third-party-import
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A class that creates junctions in temp directories on Windows.
+
+Only use this class on Windows, do not use on other platforms. Other platforms
+support longer paths than Windows, and also support symlinks. Windows only
+supports junctions (directory symlinks).
+
+Junctions are useful if you need to shorten a long path. A long path is one that
+is at least MAX_PATH (260) letters long. This is a constant in Windows, see
+<windows.h> and API documentation for file-handling functions such as
+CreateFileA.
+"""
+
+import os
+import subprocess
+import tempfile
+
+
+class JunctionCreationError(Exception):
+ """Raised when TempJunction fails to create an NTFS junction."""
+
+ def __init__(self, path, target, stdout):
+ Exception.__init__(
+ self,
+ "Could not create junction \"%s\" -> \"%s\"\nError from mklink:\n%s" %
+ (path, target, stdout))
+
+
+class TempJunction(object):
+ r"""Junction in a temp directory.
+
+ This object creates a temp directory and a junction under it. The junction
+ points to a user-specified path.
+
+ Use this object if you want to write files under long paths (absolute path at
+ least MAX_PATH (260) chars long). Pass the directory you want to "shorten" as
+ the initializer's argument. This object will create a junction under a shorter
+ path, that points to the long directory. The combined path of the junction and
+ files under it are more likely to be short than the original paths were.
+
+ Usage example:
+ with TempJunction("C:/some/long/path") as junc:
+ # `junc` created a temp directory and a junction in it that points to
+ # \\?\C:\some\long\path, and is itself shorter than that
+ shortpath = os.path.join(junc, "file.txt")
+ with open(shortpath, "w") as f:
+ ...do something with f...
+ # `junc` deleted the junction and its parent temp directory upon leaving
+ # the `with` statement's body
+ ...do something else...
+ """
+
+ def __init__(self, junction_target, testonly_mkdtemp=None):
+ """Initialize this object.
+
+ Args:
+ junction_target: string; an absolute Windows path; the __enter__ method
+ creates a junction that points to this path
+ testonly_mkdtemp: function(); for testing only; a custom function that
+ returns a temp directory path, you can use it to mock out
+ tempfile.mkdtemp
+ """
+ self._target = os.path.normpath(junction_target)
+ self._junction = None
+ self._mkdtemp = testonly_mkdtemp or tempfile.mkdtemp
+
+ def __enter__(self):
+ """Creates a temp directory and a junction in it, pointing to self._target.
+
+ This method is automatically called upon entering a `with` statement's body.
+
+ Returns:
+ The full path to the junction.
+ Raises:
+ JunctionCreationError: if `mklink` fails to create a junction
+ """
+ result = os.path.normpath(os.path.join(self._mkdtemp(), "j"))
+ proc = subprocess.Popen(
+ "cmd.exe /C mklink /J \"%s\" \"\\\\?\\%s\"" %
+ (result, os.path.normpath(self._target)),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ exitcode = proc.wait()
+ if exitcode != 0:
+ stdout = proc.communicate()[0]
+ raise JunctionCreationError(result, self._target, stdout)
+ self._junction = result
+ return result
+
+ def __exit__(self, unused_type, unused_value, unused_traceback):
+ """Deletes the junction and its parent directory.
+
+ This method is automatically called upon leaving a `with` statement's body.
+
+ Args:
+ unused_type: unused
+ unused_value: unused
+ unused_traceback: unused
+ """
+ if self._junction:
+ os.rmdir(self._junction)
+ os.rmdir(os.path.dirname(self._junction))
diff --git a/tools/android/junction_test.py b/tools/android/junction_test.py
new file mode 100644
index 0000000..0ee1b08
--- /dev/null
+++ b/tools/android/junction_test.py
@@ -0,0 +1,77 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for TempJunction."""
+
+import os
+import unittest
+
+from src.test.py.bazel import test_base
+from tools.android import junction
+
+
+class JunctionTest(test_base.TestBase):
+ """Unit tests for junction.py."""
+
+ def testCreateJunction(self):
+
+ def tempdir():
+ return self.ScratchDir("junc temp")
+
+ target = self.ScratchDir("junc target")
+ # Make the `target` path a non-normalized Windows path with a space in it.
+ # TempJunction should still work.
+ target = os.path.dirname(target) + "/junc target"
+ juncpath = None
+ with junction.TempJunction(target, testonly_mkdtemp=tempdir) as j:
+ juncpath = j
+ # Ensure that `j` created the junction.
+ self.assertTrue(os.path.exists(target))
+ self.assertTrue(os.path.exists(juncpath))
+ # Create a file under the junction.
+ filepath = os.path.join(juncpath, "file.txt")
+ with open(filepath, "w") as f:
+ f.write("hello")
+ # Ensure we can reach the file via the junction and the target directory.
+ self.assertTrue(os.path.exists(os.path.join(target, "file.txt")))
+ self.assertTrue(os.path.exists(os.path.join(juncpath, "file.txt")))
+ # Ensure that after the `with` block the junction and temp directories no
+ # longer exist, but we can still reach the file via the target directory.
+ self.assertTrue(os.path.exists(os.path.join(target, "file.txt")))
+ self.assertFalse(os.path.exists(os.path.join(juncpath, "file.txt")))
+ self.assertFalse(os.path.exists(juncpath))
+ self.assertFalse(os.path.exists(os.path.dirname(juncpath)))
+
+ def testCannotCreateJunction(self):
+
+ def tempdir():
+ return self.ScratchDir("junc temp")
+
+ target = self.ScratchDir("junc target")
+ # Make the `target` path a non-normalized Windows path with a space in it.
+ # TempJunction should still work.
+ target = os.path.dirname(target) + "/junc target"
+ with junction.TempJunction(target, testonly_mkdtemp=tempdir) as j:
+ self.assertTrue(os.path.exists(j))
+ try:
+ # Ensure that TempJunction raises a JunctionCreationError if it cannot
+ # create a junction. In this case the junction already exists in that
+ # directory.
+ with junction.TempJunction(target, testonly_mkdtemp=tempdir) as _:
+ self.fail("Expected exception")
+ except junction.JunctionCreationError:
+ pass # expected
+
+
+if __name__ == "__main__":
+ unittest.main()
