Remove xz compression from Bazel's build in pkg_tar.
- clean up Python 2/3 migration hacks
- https://github.com/bazelbuild/bazel/issues/11183
RELNOTES:
Dropped fragile xz support from built in pkg_tar. Users requiring xz
compression should switch to bazlebuild/rules_pkg.
PiperOrigin-RevId: 371394411
diff --git a/tools/build_defs/pkg/BUILD b/tools/build_defs/pkg/BUILD
index bdd8347..9ead0c2 100644
--- a/tools/build_defs/pkg/BUILD
+++ b/tools/build_defs/pkg/BUILD
@@ -42,10 +42,8 @@
],
data = [":archive_testdata"],
python_version = "PY3",
- srcs_version = "PY2AND3",
+ srcs_version = "PY3",
tags = [
- # archive.py requires xzcat, which is not available by default on Mac
- "noci",
# TODO(laszlocsomor): fix on Windows or describe why it cannot pass.
"no_windows",
],
@@ -56,7 +54,8 @@
name = "path_test",
srcs = ["path_test.py"],
data = ["path.bzl"],
- srcs_version = "PY2AND3",
+ python_version = "PY3",
+ srcs_version = "PY3",
)
py_binary(
@@ -103,7 +102,6 @@
"",
".gz",
".bz2",
- ".xz", # This will breaks if xzcat is not installed
]]
[pkg_tar(
@@ -114,7 +112,6 @@
"",
"gz",
"bz2",
- "xz",
]]
pkg_tar(
@@ -207,18 +204,13 @@
":test-tar-inclusion-.tar",
":test-tar-inclusion-bz2.tar",
":test-tar-inclusion-gz.tar",
- ":test-tar-inclusion-xz.tar",
":test-tar-mtime.tar",
":test-tar-strip_prefix-dot.tar",
":test-tar-strip_prefix-empty.tar",
":test-tar-strip_prefix-etc.tar",
":test-tar-strip_prefix-none.tar",
- ":test-tar-xz.tar.xz",
- ":titi_test_all.changes",
],
tags = [
- # archive.py requires xzcat, which is not available by default on Mac
- "noci",
# TODO(laszlocsomor): fix on Windows or describe why it cannot pass.
"no_windows",
],
diff --git a/tools/build_defs/pkg/archive.py b/tools/build_defs/pkg/archive.py
index d786624..8d6e569 100644
--- a/tools/build_defs/pkg/archive.py
+++ b/tools/build_defs/pkg/archive.py
@@ -21,7 +21,6 @@
import gzip
import io
import os
-import subprocess
import tarfile
import six
@@ -46,7 +45,7 @@
Args:
name: the tar file name.
- compression: compression type: bzip2, bz2, gz, tgz, xz, lzma.
+ compression: compression type: bzip2, bz2, gz, tgz.
root_directory: virtual root to prepend to elements in the archive.
default_mtime: default mtime to use for elements in the archive.
May be an integer or the value 'portable' to use the date
@@ -58,8 +57,6 @@
else:
mode = 'w:'
self.gz = compression in ['tgz', 'gz']
- # Support xz compression through xz... until we can use Py3
- self.xz = compression in ['xz', 'lzma']
self.name = name
self.root_directory = six.ensure_str(root_directory).rstrip('/')
@@ -283,36 +280,16 @@
compression = 'gz'
elif compression == 'bzip2':
compression = 'bz2'
- elif compression == 'lzma':
- compression = 'xz'
- elif compression not in ['gz', 'bz2', 'xz']:
+ elif compression not in ['gz', 'bz2']:
compression = ''
- if compression == 'xz':
- # Python 2 does not support lzma, our py3 support is terrible so let's
- # just hack around.
- # Note that we buffer the file in memory and it can have an important
- # memory footprint but it's probably fine as we don't use them for really
- # large files.
- # TODO(dmarting): once our py3 support gets better, compile this tools
- # with py3 for proper lzma support.
- if subprocess.call('which xzcat', shell=True, stdout=subprocess.PIPE):
- raise self.Error('Cannot handle .xz and .lzma compression: '
- 'xzcat not found.')
- p = subprocess.Popen('cat %s | xzcat' % tar,
- shell=True,
- stdout=subprocess.PIPE)
- f = io.BytesIO(p.stdout.read())
- p.wait()
- intar = tarfile.open(fileobj=f, mode='r:')
+ if compression in ['gz', 'bz2']:
+ # prevent performance issues due to accidentally-introduced seeks
+ # during intar traversal by opening in "streaming" mode. gz, bz2
+ # are supported natively by python 2.7 and 3.x
+ inmode = 'r|' + six.ensure_str(compression)
else:
- if compression in ['gz', 'bz2']:
- # prevent performance issues due to accidentally-introduced seeks
- # during intar traversal by opening in "streaming" mode. gz, bz2
- # are supported natively by python 2.7 and 3.x
- inmode = 'r|' + six.ensure_str(compression)
- else:
- inmode = 'r:' + six.ensure_str(compression)
- intar = tarfile.open(name=tar, mode=inmode)
+ inmode = 'r:' + six.ensure_str(compression)
+ intar = tarfile.open(name=tar, mode=inmode)
for tarinfo in intar:
if name_filter is None or name_filter(tarinfo.name):
if not self.preserve_mtime:
@@ -370,12 +347,3 @@
# Close the gzip file object if necessary.
if self.fileobj:
self.fileobj.close()
- if self.xz:
- # Support xz compression through xz... until we can use Py3
- if subprocess.call('which xz', shell=True, stdout=subprocess.PIPE):
- raise self.Error('Cannot handle .xz and .lzma compression: '
- 'xz not found.')
- subprocess.call(
- 'mv {0} {0}.d && xz -z {0}.d && mv {0}.d.xz {0}'.format(self.name),
- shell=True,
- stdout=subprocess.PIPE)
diff --git a/tools/build_defs/pkg/archive_test.py b/tools/build_defs/pkg/archive_test.py
index 9ab79c4..a300f1d 100644
--- a/tools/build_defs/pkg/archive_test.py
+++ b/tools/build_defs/pkg/archive_test.py
@@ -23,9 +23,6 @@
import tarfile
import unittest
-# Do not edit this line. Copybara replaces it with PY2 migration helper.
-import six
-
from tools.build_defs.pkg import archive
from tools.build_defs.pkg import testenv
@@ -83,13 +80,13 @@
def assertSimpleFileContent(self, names):
with archive.TarFileWriter(self.tempfile) as f:
for n in names:
- f.add_file(n, content=n)
+ f.add_file(n, content=n.encode("utf-8"))
content = ([{
"name": "."
}] + [{
"name": n,
- "size": len(six.ensure_binary(n, "utf-8")),
- "data": six.ensure_binary(n, "utf-8")
+ "size": len(n.encode("utf-8")),
+ "data": n.encode("utf-8")
} for n in names])
self.assertTarFileContent(self.tempfile, content)
@@ -170,7 +167,7 @@
{"name": "./a", "data": b"a"},
{"name": "./ab", "data": b"ab"},
]
- for ext in ["", ".gz", ".bz2", ".xz"]:
+ for ext in ["", ".gz", ".bz2"]:
with archive.TarFileWriter(self.tempfile) as f:
f.add_tar(os.path.join(testenv.TESTDATA_PATH, "tar_test.tar" + ext),
name_filter=lambda n: n != "./b")
diff --git a/tools/build_defs/pkg/build_test.sh b/tools/build_defs/pkg/build_test.sh
index 0be65f5..99c4c87 100755
--- a/tools/build_defs/pkg/build_test.sh
+++ b/tools/build_defs/pkg/build_test.sh
@@ -86,7 +86,7 @@
./usr/titi
./usr/bin/
./usr/bin/java -> /path/to/bin/java"
- for i in "" ".gz" ".bz2" ".xz"; do
+ for i in "" ".gz" ".bz2"; do
assert_content "test-tar-${i:1}.tar$i"
# Test merging tar files
# We pass a second argument to not test for user and group
diff --git a/tools/build_defs/pkg/path_test.py b/tools/build_defs/pkg/path_test.py
index 606f7b2..934b61a 100644
--- a/tools/build_defs/pkg/path_test.py
+++ b/tools/build_defs/pkg/path_test.py
@@ -16,7 +16,9 @@
import imp
import unittest
-pkg_bzl = imp.load_source('pkg_bzl', 'tools/build_defs/pkg/path.bzl')
+pkg_bzl = imp.load_source(
+ 'pkg_bzl',
+ 'tools/build_defs/pkg/path.bzl')
class File(object):
diff --git a/tools/build_defs/pkg/pkg.bzl b/tools/build_defs/pkg/pkg.bzl
index 11fce9b..3cbd8f1 100644
--- a/tools/build_defs/pkg/pkg.bzl
+++ b/tools/build_defs/pkg/pkg.bzl
@@ -17,7 +17,7 @@
load("//tools/config:common_settings.bzl", "BuildSettingInfo")
# Filetype to restrict inputs
-tar_filetype = [".tar", ".tar.gz", ".tgz", ".tar.xz", ".tar.bz2"]
+tar_filetype = [".tar", ".tar.gz", ".tgz", ".tar.bz2"]
def _remap(remap_paths, path):
"""If path starts with a key in remap_paths, rewrite it."""
diff --git a/tools/build_defs/pkg/testdata/tar_test.tar.xz b/tools/build_defs/pkg/testdata/tar_test.tar.xz
deleted file mode 100644
index 1ea3c8b..0000000
--- a/tools/build_defs/pkg/testdata/tar_test.tar.xz
+++ /dev/null
Binary files differ
