Add support for .ar archives (and .deb files)
This implements #[15130](https://github.com/bazelbuild/bazel/issues/15130).
As I was updating the docs for .ar and .deb formats, I also addressed some previous formats that had been added but not propagated through to all the documentation places.
Closes #15132.
PiperOrigin-RevId: 439569440
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java
new file mode 100644
index 0000000..dc4ac44
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java
@@ -0,0 +1,81 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.io.ByteStreams;
+import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import org.apache.commons.compress.archivers.ar.ArArchiveEntry;
+import org.apache.commons.compress.archivers.ar.ArArchiveInputStream;
+
+/**
+ * Opens a .ar archive file. It ignores the prefix setting because these archives cannot contain
+ * directories.
+ */
+public class ArFunction implements Decompressor {
+
+ public static final Decompressor INSTANCE = new ArFunction();
+
+ // This is the same value as picked for .tar files, which appears to have worked well.
+ private static final int BUFFER_SIZE = 32 * 1024;
+
+ private InputStream getDecompressorStream(DecompressorDescriptor descriptor) throws IOException {
+ return new BufferedInputStream(
+ new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE);
+ }
+ ;
+
+ @Override
+ public Path decompress(DecompressorDescriptor descriptor)
+ throws InterruptedException, IOException {
+ if (Thread.interrupted()) {
+ throw new InterruptedException();
+ }
+
+ try (InputStream decompressorStream = getDecompressorStream(descriptor)) {
+ ArArchiveInputStream arStream = new ArArchiveInputStream(decompressorStream);
+ ArArchiveEntry entry;
+ while ((entry = arStream.getNextArEntry()) != null) {
+ Path filePath = descriptor.repositoryPath().getRelative(entry.getName());
+ filePath.getParentDirectory().createDirectoryAndParents();
+ if (entry.isDirectory()) {
+ // ar archives don't contain any directory information, so this should never
+ // happen
+ continue;
+ } else {
+ // We do not have to worry about symlinks in .ar files - it's not supported
+ // by the .ar file format.
+ try (OutputStream out = filePath.getOutputStream()) {
+ ByteStreams.copy(arStream, out);
+ }
+ filePath.chmod(entry.getMode());
+ // entry.getLastModified() appears to be in seconds, so we need to convert
+ // it into milliseconds for setLastModifiedTime
+ filePath.setLastModifiedTime(entry.getLastModified() * 1000L);
+ }
+ if (Thread.interrupted()) {
+ throw new InterruptedException();
+ }
+ }
+ }
+
+ return descriptor.repositoryPath();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java
index 49ff20b..a651c2c 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java
@@ -107,11 +107,13 @@
return TarZstFunction.INSTANCE;
} else if (baseName.endsWith(".tar.bz2")) {
return TarBz2Function.INSTANCE;
+ } else if (baseName.endsWith(".ar") || baseName.endsWith(".deb")) {
+ return ArFunction.INSTANCE;
} else {
throw new RepositoryFunctionException(
Starlark.errorf(
"Expected a file with a .zip, .jar, .war, .aar, .tar, .tar.gz, .tgz, .tar.xz, .txz,"
- + " .tar.zst, .tzst, or .tar.bz2 suffix (got %s)",
+ + " .tar.zst, .tzst, .tar.bz2, .ar or .deb suffix (got %s)",
archivePath),
Transience.PERSISTENT);
}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java
index 217f8b0..1ef1d77 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java
@@ -825,7 +825,8 @@
+ " By default, the archive type is determined from the file extension of"
+ " the URL."
+ " If the file has no extension, you can explicitly specify either \"zip\","
- + " \"jar\", \"war\", \"aar\", \"tar.gz\", \"tgz\", \"tar.bz2\", or \"tar.xz\""
+ + " \"jar\", \"war\", \"aar\", \"tar\", \"tar.gz\", \"tgz\", \"tar.xz\","
+ + " \"txz\", \".tar.zst\", \".tzst\", \"tar.bz2\", \".ar\", or \".deb\""
+ " here."),
@Param(
name = "stripPrefix",
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java
new file mode 100644
index 0000000..96077f0
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java
@@ -0,0 +1,88 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.devtools.build.lib.testutil.TestConstants;
+import com.google.devtools.build.lib.testutil.TestUtils;
+import com.google.devtools.build.lib.unix.UnixFileSystem;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.DigestHashFunction;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.JavaIoFileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.runfiles.Runfiles;
+import java.io.File;
+import java.io.IOException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests decompressing archives. */
+@RunWith(JUnit4.class)
+public class ArFunctionTest {
+ /*
+ * .ar archive created with ar cr test_files.ar archived_first.txt archived_second.md
+ * The files contain short UTF-8 encoded strings.
+ */
+ private static final String ARCHIVE_NAME = "test_files.ar";
+ private static final String PATH_TO_TEST_ARCHIVE =
+ "/com/google/devtools/build/lib/bazel/repository/";
+ private static final String FIRST_FILE_NAME = "archived_first.txt";
+ private static final String SECOND_FILE_NAME = "archived_second.md";
+
+ @Test
+ public void testDecompress() throws Exception {
+ Path outputDir = decompress(createDescriptorBuilder());
+
+ assertThat(outputDir.exists()).isTrue();
+ Path firstFile = outputDir.getRelative(FIRST_FILE_NAME);
+ assertThat(firstFile.exists()).isTrue();
+ // There are 20 bytes in the content "this is test file 1"
+ assertThat(firstFile.getFileSize()).isEqualTo(20);
+ assertThat(firstFile.isSymbolicLink()).isFalse();
+
+ Path secondFile = outputDir.getRelative(SECOND_FILE_NAME);
+ assertThat(secondFile.exists()).isTrue();
+ // There are 20 bytes in the content "this is the second test file"
+ assertThat(secondFile.getFileSize()).isEqualTo(29);
+ assertThat(secondFile.isSymbolicLink()).isFalse();
+ }
+
+ private Path decompress(DecompressorDescriptor.Builder descriptorBuilder) throws Exception {
+ descriptorBuilder.setDecompressor(ArFunction.INSTANCE);
+ return new ArFunction().decompress(descriptorBuilder.build());
+ }
+
+ private DecompressorDescriptor.Builder createDescriptorBuilder() throws IOException {
+ // This was cribbed from TestArchiveDescriptor
+ FileSystem testFS =
+ OS.getCurrent() == OS.WINDOWS
+ ? new JavaIoFileSystem(DigestHashFunction.SHA256)
+ : new UnixFileSystem(DigestHashFunction.SHA256, /*hashAttributeName=*/ "");
+
+ // do not rely on TestConstants.JAVATESTS_ROOT end with slash, but ensure separators
+ // are not duplicated
+ String path =
+ (TestConstants.JAVATESTS_ROOT + PATH_TO_TEST_ARCHIVE + ARCHIVE_NAME).replace("//", "/");
+ Path tarballPath = testFS.getPath(Runfiles.create().rlocation(path));
+
+ Path workingDir = testFS.getPath(new File(TestUtils.tmpDir()).getCanonicalPath());
+ Path outDir = workingDir.getRelative("out");
+
+ return DecompressorDescriptor.builder().setRepositoryPath(outDir).setArchivePath(tarballPath);
+ }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD
index 2397b27..f9cf05b 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD
@@ -27,6 +27,7 @@
data = [
"test_decompress_archive.tar.gz",
"test_decompress_archive.zip",
+ "test_files.ar",
],
deps = [
"//src/main/java/com/google/devtools/build/lib/bazel/repository",
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java
index 032da6f..f457f2b 100644
--- a/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java
+++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java
@@ -56,6 +56,10 @@
DecompressorDescriptor.builder().setArchivePath(path).build();
path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.bz2");
DecompressorDescriptor.builder().setArchivePath(path).build();
+ path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.ar");
+ DecompressorDescriptor.builder().setArchivePath(path).build();
+ path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.deb");
+ DecompressorDescriptor.builder().setArchivePath(path).build();
}
@Test
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar b/src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar
new file mode 100644
index 0000000..8aaddbe
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar
@@ -0,0 +1,9 @@
+!<arch>
+// 40 `
+archived_first.txt/
+archived_second.md/
+/0 0 0 0 644 20 `
+this is test file 1
+/20 0 0 0 644 29 `
+this is the second test file
+
diff --git a/tools/build_defs/repo/http.bzl b/tools/build_defs/repo/http.bzl
index 1bbd2f9..fd40a5b 100644
--- a/tools/build_defs/repo/http.bzl
+++ b/tools/build_defs/repo/http.bzl
@@ -269,7 +269,7 @@
By default, the archive type is determined from the file extension of the
URL. If the file has no extension, you can explicitly specify one of the
following: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`, `"tar.gz"`, `"tgz"`,
-`"tar.xz"`, or `tar.bz2`.""",
+`"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`, or `"deb"`.""",
),
"patches": attr.label_list(
default = [],
@@ -357,8 +357,9 @@
"""Downloads a Bazel repository as a compressed archive file, decompresses it,
and makes its targets available for binding.
-It supports the following file extensions: `"zip"`, `"jar"`, `"war"`, `"aar"`,
-`"tar"`, `"tar.gz"`, `"tgz"`, `"tar.xz"`, and `tar.bz2`.
+It supports the following file extensions: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`,
+`"tar.gz"`, `"tgz"`, `"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`,
+or `"deb"`.
Examples:
Suppose the current repository contains the source code for a chat program,
