[9.3.2] Restore server JAR recompression in `PackageZipper` (#31299) (#32199) Optimized Bazel packages now repackage the server JAR with deterministic stored entries before compressing it as `A-server.jar` in the outer archive, restoring behavior that was lost in a677940ab751388ad3e7dec323ca6e9a19b7e039. After that commit, the outer deflate stream could no longer exploit redundancy across class files and instead recompressed an already-compressed JAR. On macOS arm64, this reduces `bazel build -c opt //src:bazel` from 68.31 MiB to 57.82 MiB (a 15.4% reduction). RELNOTES: None (cherry picked from commit e4ccdd44b20e087d9ebd6b140489d6379824e159) Co-authored-by: twerth <twerth@google.com>
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/PackageZipper.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/PackageZipper.java index 165a87e..227c72a 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/zip/PackageZipper.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/PackageZipper.java
@@ -15,6 +15,7 @@ package com.google.devtools.build.zip; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Comparator.comparing; import com.google.devtools.build.zip.ZipFileEntry.Compression; import java.io.ByteArrayOutputStream; @@ -118,7 +119,7 @@ continue; } try (InputStream in = zipFile.getInputStream(ze)) { - byte[] data = readAllBytes(in); + byte[] data = in.readAllBytes(); entries.add(new PackageEntry("embedded_tools/" + ze.getName(), data)); } } @@ -133,7 +134,7 @@ if (te.isDirectory()) { continue; } - byte[] data = readAllBytes(tarIn); + byte[] data = tarIn.readAllBytes(); entries.add(new PackageEntry(te.getName(), data)); } } @@ -144,6 +145,9 @@ Collections.sort(entries); + byte[] serverJarBytes = + fastCompression ? Files.readAllBytes(serverJar) : makeUncompressedServerJar(serverJar); + if (outputPath.getParent() != null) { Files.createDirectories(outputPath.getParent()); } @@ -152,7 +156,7 @@ ZipWriter writer = new ZipWriter(fos, UTF_8, true); // 1. Mandatory first entry: A-server.jar (for Bazel client JVM bootstrapper) - writeBytesEntry(writer, "A-server.jar", Files.readAllBytes(serverJar), compressionLevel); + writeBytesEntry(writer, "A-server.jar", serverJarBytes, compressionLevel); // 2. Sorted intermediate package entries (including build-label.txt, embedded_tools/, // platforms/, tools, binaries) @@ -174,12 +178,24 @@ } } - private static byte[] readAllBytes(InputStream in) throws IOException { + /** + * Repackages the server JAR with uncompressed entries so that compressing it as a single entry in + * the outer archive can exploit redundancy across class files. + */ + private static byte[] makeUncompressedServerJar(Path serverJar) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - byte[] buffer = new byte[65536]; - int n; - while ((n = in.read(buffer)) > 0) { - out.write(buffer, 0, n); + try (ZipFile zipFile = new ZipFile(serverJar.toFile(), UTF_8); + ZipWriter writer = new ZipWriter(out, UTF_8, true)) { + List<? extends ZipEntry> entries = + zipFile.stream() + .filter(entry -> !entry.isDirectory()) + .sorted(comparing(ZipEntry::getName)) + .toList(); + for (ZipEntry entry : entries) { + try (InputStream in = zipFile.getInputStream(entry)) { + writeStoredBytesEntry(writer, entry.getName(), in.readAllBytes()); + } + } } return out.toByteArray(); } @@ -217,17 +233,38 @@ writer.closeEntry(); } + private static void writeStoredBytesEntry(ZipWriter writer, String entryName, byte[] rawBytes) + throws IOException { + CRC32 crc32 = new CRC32(); + crc32.update(rawBytes); + + ZipFileEntry entry = new ZipFileEntry(entryName); + entry.setTime(ZipUtil.DOS_EPOCH); + entry.setVersion((short) 20); + entry.setMethod(Compression.STORED); + entry.setSize(rawBytes.length); + entry.setCompressedSize(rawBytes.length); + entry.setCrc(crc32.getValue()); + + writer.putNextEntry(entry); + writer.write(rawBytes); + writer.closeEntry(); + } + private static byte[] deflate(byte[] data, int compressionLevel) { Deflater deflater = new Deflater(compressionLevel, true); - deflater.setInput(data); - deflater.finish(); - ByteArrayOutputStream out = new ByteArrayOutputStream(data.length); - byte[] buf = new byte[65536]; - while (!deflater.finished()) { - int count = deflater.deflate(buf); - out.write(buf, 0, count); + try { + deflater.setInput(data); + deflater.finish(); + ByteArrayOutputStream out = new ByteArrayOutputStream(data.length); + byte[] buf = new byte[65536]; + while (!deflater.finished()) { + int count = deflater.deflate(buf); + out.write(buf, 0, count); + } + return out.toByteArray(); + } finally { + deflater.end(); } - deflater.end(); - return out.toByteArray(); } }
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/PackageZipperTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/PackageZipperTest.java index 926c1c7..e976fa0 100644 --- a/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/PackageZipperTest.java +++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/PackageZipperTest.java
@@ -18,6 +18,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; @@ -56,9 +57,19 @@ try (OutputStream out = Files.newOutputStream(serverJar); ZipOutputStream jar = new ZipOutputStream(out)) { + jar.putNextEntry(new ZipEntry("z.txt")); + jar.write("shared-prefix-z".getBytes(UTF_8)); + jar.closeEntry(); jar.putNextEntry(new ZipEntry("build-data.properties")); jar.write("build.label=1.2.3\n".getBytes(UTF_8)); jar.closeEntry(); + jar.putNextEntry(new ZipEntry("a.txt")); + jar.write("shared-prefix-a".getBytes(UTF_8)); + jar.closeEntry(); + jar.putNextEntry(new ZipEntry("empty-dir/")); + jar.closeEntry(); + jar.putNextEntry(new ZipEntry("empty.txt")); + jar.closeEntry(); } Files.write(installBaseKey, "0123456789abcdef".getBytes(UTF_8)); } @@ -87,4 +98,55 @@ byte[] archive = Files.readAllBytes(outputZip); assertThat(ZipUtil.get32(archive, LOCAL_HEADER_MOD_TIME_OFFSET)).isEqualTo(DOS_1980_01_01); } + + @Test + public void optimizedBundleRepackagesServerJarWithStoredEntries() throws IOException { + PackageZipper.main( + new String[] {outputZip.toString(), serverJar.toString(), installBaseKey.toString()}); + + Path nestedServerJar = tmpDir.resolve("nested-server.jar"); + try (ZipFile zip = new ZipFile(outputZip.toFile(), UTF_8)) { + ZipEntry serverJarEntry = zip.getEntry("A-server.jar"); + assertThat(serverJarEntry).isNotNull(); + assertThat(serverJarEntry.getMethod()).isEqualTo(ZipEntry.DEFLATED); + try (InputStream in = zip.getInputStream(serverJarEntry)) { + Files.copy(in, nestedServerJar); + } + } + + List<String> names = new ArrayList<>(); + try (ZipFile origJar = new ZipFile(serverJar.toFile(), UTF_8); + ZipFile nestedJar = new ZipFile(nestedServerJar.toFile(), UTF_8)) { + for (ZipEntry entry : Collections.list(nestedJar.entries())) { + names.add(entry.getName()); + assertThat(entry.getMethod()).isEqualTo(ZipEntry.STORED); + assertThat(ZipUtil.unixToDosTime(entry.getTime())).isEqualTo(DOS_1980_01_01); + ZipEntry origEntry = origJar.getEntry(entry.getName()); + assertThat(origEntry).isNotNull(); + try (InputStream in = nestedJar.getInputStream(entry); + InputStream origIn = origJar.getInputStream(origEntry)) { + assertThat(in.readAllBytes()).isEqualTo(origIn.readAllBytes()); + } + } + } + assertThat(names) + .containsExactly("a.txt", "build-data.properties", "empty.txt", "z.txt") + .inOrder(); + } + + @Test + public void fastBundleKeepsServerJarAsIs() throws IOException { + PackageZipper.main( + new String[] { + "--fast", outputZip.toString(), serverJar.toString(), installBaseKey.toString() + }); + + try (ZipFile zip = new ZipFile(outputZip.toFile(), UTF_8)) { + ZipEntry serverJarEntry = zip.getEntry("A-server.jar"); + assertThat(serverJarEntry).isNotNull(); + try (InputStream in = zip.getInputStream(serverJarEntry)) { + assertThat(in.readAllBytes()).isEqualTo(Files.readAllBytes(serverJar)); + } + } + } }