[9.3.0] More Windows zip fixes (#31148) Original commits: * 33360736f07ab6e383fd9ddd009544379692c40e * a677940ab751388ad3e7dec323ca6e9a19b7e039 --------- Co-authored-by: tedx <tedx@google.com> Co-authored-by: twerth <twerth@google.com>
diff --git a/src/BUILD b/src/BUILD index cf6a823..fe4891e 100644 --- a/src/BUILD +++ b/src/BUILD
@@ -6,7 +6,9 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") load("//src:release_archive.bzl", "minimum_java_compilation_runtime_filegroup", "minimum_java_runtime_filegroup", "release_archive") +load(":bazel_binary.bzl", "bazel_binary") load(":embedded_tools.bzl", "srcsfile") +load(":package_zip.bzl", "package_zip") load(":rule_size_test.bzl", "rule_size_test") package(default_applicable_licenses = ["@io_bazel//:license"]) @@ -287,20 +289,9 @@ "_nojdk", ]] -[genrule( +[package_zip( name = "package-zip" + suffix, - # This script assumes the following arg order: - # 1) embedded tools zip (if it exists) - # 2) the deploy jar - # 3) the install base key - # 4) the platforms archive - # 5) everything else to be bundled - # Ensure we haven't forgotten to add all package-zip items as dependencies - # of :install_base_key-file, otherwise bazel won't correctly reextract - # modified files. srcs = ([":embedded_tools" + suffix + ".zip"] if embed else []) + [ - "//src/main/java/com/google/devtools/build/lib/bazel:BazelServer_deploy.jar", - "install_base_key" + suffix, "//:platforms-srcs", # Non-ordered items follow: "//src/main/java/net/starlark/java/eval:cpu_profiler", @@ -313,14 +304,15 @@ "//src/main/tools:daemonize", ], }), - outs = ["package" + suffix + ".zip"], - cmd = "$(location :package_bazel_on_host_platform) $@ " + ("" if embed else "''") + " $(SRCS)", - tools = [":package_bazel_on_host_platform"], -) for suffix, embed in [ - ("_jdk_allmodules", True), - ("_jdk_minimal", True), - ("_notools", False), - ("_nojdk", True), + out = "package" + suffix + ".zip", + dev_build = dev_build, + install_base_key = "install_base_key" + suffix, + server_jar = "//src/main/java/com/google/devtools/build/lib/bazel:BazelServer_deploy.jar", +) for suffix, embed, dev_build in [ + ("_jdk_allmodules", True, True), + ("_jdk_minimal", True, False), + ("_notools", False, False), + ("_nojdk", True, False), ]] genrule( @@ -334,20 +326,11 @@ }), ) -[genrule( +[bazel_binary( name = "bazel-bin" + suffix, - srcs = [ - "//src/main/cpp:client", - "package-zip" + jdk, - ], - outs = ["bazel" + suffix], - # In msys, a file path without .exe suffix(say foo), refers to a file with .exe - # suffix(say foo.exe), if foo.exe exists and foo doesn't. So, on windows, we - # need to remove bazel.exe first, so that cat to bazel won't fail. - cmd = "rm -f $@ $@.unaligned; cat $(location //src/main/cpp:client) $(location :package-zip" + jdk + ") > $@.unaligned && $(location //src/java_tools/singlejar/java/com/google/devtools/build/zip:adjust_sfx) $@.unaligned $@ && rm -f $@.unaligned && chmod a+x $@", - executable = 1, - output_to_bindir = 1, - tools = ["//src/java_tools/singlejar/java/com/google/devtools/build/zip:adjust_sfx"], + out = "bazel" + suffix, + client = "//src/main/cpp:client", + package_zip = "package-zip" + jdk, visibility = [ "//:__pkg__", # For distribution archive lockfile generation "//scripts:__pkg__", # For bash completion generation @@ -365,9 +348,8 @@ ("_nojdk", "_nojdk"), ]] -# Build an executable named `bazel.exe`. +# Build an executable named `bazel.exe` (and suffixes). # Works on all platforms but only makes sense on Windows. -# Workaround for non-configurability of genrule's `outs` attribute. [genrule( name = "bazel-bin" + suffix + ".exe", srcs = [":bazel-bin" + suffix], @@ -432,6 +414,7 @@ "//src/tools/launcher:srcs", "//src/tools/one_version:srcs", "//src/tools/remote:srcs", + "//src/tools/simple_catter:srcs", "//src/tools/singlejar:srcs", "//src/tools/starlark/java/com/google/devtools/starlark/common:srcs", "//src/tools/workspacelog:srcs",
diff --git a/src/bazel_binary.bzl b/src/bazel_binary.bzl new file mode 100644 index 0000000..325563d --- /dev/null +++ b/src/bazel_binary.bzl
@@ -0,0 +1,69 @@ +# Copyright 2026 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Custom rule for packaging Bazel binary.""" + +def _bazel_binary_impl(ctx): + client = ctx.file.client + package_zip = ctx.file.package_zip + cat_binary = ctx.executable._cat_binary + adjust_sfx = ctx.executable._adjust_sfx + output = ctx.outputs.out + + unaligned = ctx.actions.declare_file(output.basename + ".unaligned") + + # Concatenate client and zip + ctx.actions.run( + inputs = [client, package_zip], + outputs = [unaligned], + executable = cat_binary, + arguments = [client.path, package_zip.path, unaligned.path], + mnemonic = "ConcatClientAndZip", + progress_message = "Concatenating client and zip for %{ctx.label.name}", + ) + + # Adjust SFX + ctx.actions.run( + inputs = [unaligned], + outputs = [output], + executable = adjust_sfx, + arguments = [unaligned.path, output.path], + mnemonic = "AdjustSfx", + progress_message = "Adjusting SFX for %{ctx.label.name}", + ) + + return [DefaultInfo( + executable = output, + files = depset([output]), + )] + +bazel_binary = rule( + implementation = _bazel_binary_impl, + attrs = { + "client": attr.label(allow_single_file = True, mandatory = True), + "package_zip": attr.label(allow_single_file = True, mandatory = True), + "out": attr.output(mandatory = True), + "_cat_binary": attr.label( + default = Label("//src/tools/simple_catter:simple_catter"), + executable = True, + cfg = "exec", + ), + "_adjust_sfx": attr.label( + default = Label("//src/java_tools/singlejar/java/com/google/devtools/build/zip:adjust_sfx"), + executable = True, + cfg = "exec", + ), + }, + executable = True, +)
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/BUILD b/src/java_tools/singlejar/java/com/google/devtools/build/zip/BUILD index 4e72105..ff70bbc 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/zip/BUILD +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/BUILD
@@ -25,7 +25,10 @@ name = "zip", srcs = glob(["*.java"]), visibility = ["//visibility:public"], - deps = ["//third_party:jsr305"], + deps = [ + "//third_party:apache_commons_compress", + "//third_party:jsr305", + ], ) # Adjusts zip offsets in self-extracting archive (like zip -qA, but out-of-place). @@ -35,3 +38,11 @@ visibility = ["//src:__pkg__"], runtime_deps = [":zip"], ) + +# Hermetic packaging tool for Bazel release archives. +java_binary( + name = "package_zipper", + main_class = "com.google.devtools.build.zip.PackageZipper", + visibility = ["//src:__pkg__"], + runtime_deps = [":zip"], +)
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/PackageZipper.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/PackageZipper.java new file mode 100644 index 0000000..b43a459 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/PackageZipper.java
@@ -0,0 +1,231 @@ +// Copyright 2026 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.devtools.build.zip.ZipFileEntry.Compression; +import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.zip.CRC32; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; + +/** + * Hermetic packaging tool for Bazel release archives (package.zip). + * + * <p>Guarantees required archive entry ordering (A-server.jar first, sorted files, install_base_key + * last), normalized 1980-01-01 timestamps, automatic unpacking of embedded_tools zips and platform + * tar archives, generation of build-label.txt from server jar properties, and synchronous Win32 + * kernel flushing (FileChannel.force(true)) before process termination. + */ +public final class PackageZipper { + private PackageZipper() {} + + private static final class PackageEntry implements Comparable<PackageEntry> { + final String name; + final byte[] data; + + PackageEntry(String name, byte[] data) { + this.name = name; + this.data = data; + } + + @Override + public int compareTo(PackageEntry other) { + return this.name.compareTo(other.name); + } + } + + public static void main(String[] args) throws IOException { + boolean fastCompression = false; + List<String> positionalArgs = new ArrayList<>(); + for (String arg : args) { + if (arg.equals("--fast")) { + fastCompression = true; + } else { + positionalArgs.add(arg); + } + } + + if (positionalArgs.size() < 3) { + System.err.println( + "Usage: PackageZipper [--fast] <output_zip> <server_jar> <install_base_key> [file...]"); + System.exit(1); + } + + Path outputPath = Path.of(positionalArgs.get(0)); + Path serverJar = Path.of(positionalArgs.get(1)); + Path installBaseKey = Path.of(positionalArgs.get(2)); + int compressionLevel = fastCompression ? Deflater.BEST_SPEED : Deflater.BEST_COMPRESSION; + + List<Path> packageFiles = new ArrayList<>(); + for (int i = 3; i < positionalArgs.size(); i++) { + Path p = Path.of(positionalArgs.get(i)); + if (Files.isRegularFile(p)) { + packageFiles.add(p); + } + } + + String buildLabel = "no_version"; + try (ZipFile serverZip = new ZipFile(serverJar.toFile(), UTF_8)) { + ZipEntry buildDataEntry = serverZip.getEntry("build-data.properties"); + if (buildDataEntry != null) { + try (InputStream in = serverZip.getInputStream(buildDataEntry)) { + Properties props = new Properties(); + props.load(in); + String label = props.getProperty("build.label"); + if (label != null && !label.isEmpty()) { + buildLabel = label; + } + } + } + } + + List<PackageEntry> entries = new ArrayList<>(); + entries.add(new PackageEntry("build-label.txt", buildLabel.getBytes(UTF_8))); + + for (Path file : packageFiles) { + String fileName = file.getFileName().toString(); + if (fileName.startsWith("embedded_tools") && fileName.endsWith(".zip")) { + try (ZipFile zipFile = new ZipFile(file.toFile(), UTF_8)) { + List<? extends ZipEntry> zipEntries = Collections.list(zipFile.entries()); + for (ZipEntry ze : zipEntries) { + if (ze.isDirectory()) { + continue; + } + try (InputStream in = zipFile.getInputStream(ze)) { + byte[] data = readAllBytes(in); + entries.add(new PackageEntry("embedded_tools/" + ze.getName(), data)); + } + } + } + } else if (fileName.endsWith(".tar") + || fileName.endsWith(".tar.gz") + || fileName.endsWith(".tgz")) { + try (InputStream fis = Files.newInputStream(file); + TarArchiveInputStream tarIn = new TarArchiveInputStream(fis)) { + TarArchiveEntry te; + while ((te = tarIn.getNextEntry()) != null) { + if (te.isDirectory()) { + continue; + } + byte[] data = readAllBytes(tarIn); + entries.add(new PackageEntry(te.getName(), data)); + } + } + } else { + entries.add(new PackageEntry(fileName, Files.readAllBytes(file))); + } + } + + Collections.sort(entries); + + if (outputPath.getParent() != null) { + Files.createDirectories(outputPath.getParent()); + } + + try (FileOutputStream fos = new FileOutputStream(outputPath.toFile())) { + ZipWriter writer = new ZipWriter(fos, UTF_8, true); + + // 1. Mandatory first entry: A-server.jar (for Bazel client JVM bootstrapper) + writeBytesEntry(writer, "A-server.jar", Files.readAllBytes(serverJar), compressionLevel); + + // 2. Sorted intermediate package entries (including build-label.txt, embedded_tools/, + // platforms/, tools, binaries) + for (PackageEntry pe : entries) { + writeBytesEntry(writer, pe.name, pe.data, compressionLevel); + } + + // 3. Mandatory last entry: install_base_key + writeBytesEntry( + writer, "install_base_key", Files.readAllBytes(installBaseKey), compressionLevel); + + writer.finish(); + + // Explicitly force NTFS MFT EOF and dirty cache pages to commit + // synchronously before process exit. + fos.getChannel().force(true); + + writer.close(); + } + } + + private static byte[] readAllBytes(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[65536]; + int n; + while ((n = in.read(buffer)) > 0) { + out.write(buffer, 0, n); + } + return out.toByteArray(); + } + + private static void writeBytesEntry( + ZipWriter writer, String entryName, byte[] rawBytes, int compressionLevel) + throws IOException { + CRC32 crc32 = new CRC32(); + crc32.update(rawBytes); + long crc = crc32.getValue(); + + byte[] compressedBytes = deflate(rawBytes, compressionLevel); + + ZipFileEntry entry = new ZipFileEntry(entryName); + entry.setTime(315532800000L); // 1980-01-01 00:00:00 UTC + entry.setVersion((short) 20); + + if (compressedBytes.length >= rawBytes.length && rawBytes.length > 0) { + entry.setMethod(Compression.STORED); + entry.setSize(rawBytes.length); + entry.setCompressedSize(rawBytes.length); + entry.setCrc(crc); + writer.putNextEntry(entry); + writer.write(rawBytes); + } else { + entry.setMethod(Compression.DEFLATED); + entry.setSize(rawBytes.length); + entry.setCompressedSize(compressedBytes.length); + entry.setCrc(crc); + writer.putNextEntry(entry); + writer.write(compressedBytes); + } + writer.closeEntry(); + } + + private static byte[] deflate(byte[] data, int compressionLevel) { + Deflater deflater = new Deflater(compressionLevel, true); + deflater.setInput(data); + deflater.finish(); + ByteArrayOutputStream out = new ByteArrayOutputStream(data.length); + byte[] buf = new byte[65536]; + while (!deflater.finished()) { + int count = deflater.deflate(buf); + out.write(buf, 0, count); + } + deflater.end(); + return out.toByteArray(); + } +}
diff --git a/src/package_zip.bzl b/src/package_zip.bzl new file mode 100644 index 0000000..81ef7ef --- /dev/null +++ b/src/package_zip.bzl
@@ -0,0 +1,52 @@ +# Copyright 2026 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hermetic Starlark rule for building Bazel package.zip archives.""" + +def _package_zip_impl(ctx): + args = ctx.actions.args() + if ctx.attr.dev_build: + args.add("--fast") + args.add(ctx.outputs.out) + args.add(ctx.file.server_jar) + args.add(ctx.file.install_base_key) + args.add_all(ctx.files.srcs) + + ctx.actions.run( + executable = ctx.executable._package_zipper, + arguments = [args], + inputs = [ctx.file.server_jar, ctx.file.install_base_key] + ctx.files.srcs, + outputs = [ctx.outputs.out], + mnemonic = "PackageZip", + progress_message = "Building %{output}", + ) + +package_zip = rule( + implementation = _package_zip_impl, + attrs = { + "server_jar": attr.label(allow_single_file = True, mandatory = True), + "install_base_key": attr.label(allow_single_file = True, mandatory = True), + "srcs": attr.label_list(allow_files = True), + "out": attr.output(mandatory = True), + "dev_build": attr.bool( + default = False, + doc = "If True, use fast compression (level 1) for developer builds.", + ), + "_package_zipper": attr.label( + default = "//src/java_tools/singlejar/java/com/google/devtools/build/zip:package_zipper", + executable = True, + cfg = "exec", + ), + }, +)
diff --git a/src/tools/simple_catter/BUILD b/src/tools/simple_catter/BUILD new file mode 100644 index 0000000..c6ce927 --- /dev/null +++ b/src/tools/simple_catter/BUILD
@@ -0,0 +1,13 @@ +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src:__pkg__"], +) + +cc_binary( + name = "simple_catter", + srcs = ["simple_catter.cc"], + visibility = ["//visibility:public"], +)
diff --git a/src/tools/simple_catter/simple_catter.cc b/src/tools/simple_catter/simple_catter.cc new file mode 100644 index 0000000..7a1f80e --- /dev/null +++ b/src/tools/simple_catter/simple_catter.cc
@@ -0,0 +1,75 @@ +// Copyright 2026 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <fstream> +#include <iostream> +#include <string> + +/** + * This tool concatenates two input files into a single output file. + * It provides the same basic functionality as the system 'cat' command + * but is contained entirely within the Bazel repository to avoid + * relying on the host environment's 'cat' utility. + * + * Usage: simple_catter <input_file1> <input_file2> <output_file> + */ +int main(int argc, char* argv[]) { + if (argc != 4) { + std::cerr << "Usage: " << argv[0] + << " <input_file1> <input_file2> <output_file>\n"; + return 1; + } + + std::string input1_path = argv[1]; + std::string input2_path = argv[2]; + std::string output_path = argv[3]; + + std::ofstream out(output_path, std::ios::binary); + if (!out) { + std::cerr << "Error opening output file: " << output_path << "\n"; + return 1; + } + + // Copy input 1 + std::ifstream in(input1_path, std::ios::binary); + if (!in) { + std::cerr << "Error opening input file: " << input1_path << "\n"; + return 1; + } + out << in.rdbuf(); + if (!out) { + std::cerr << "Error writing to output file while copying: " << input1_path + << "\n"; + return 1; + } + in.close(); + + // Copy input 2 + in.open(input2_path, std::ios::binary); + if (!in) { + std::cerr << "Error opening input file: " << input2_path << "\n"; + return 1; + } + out << in.rdbuf(); + if (!out) { + std::cerr << "Error writing to output file while copying: " << input2_path + << "\n"; + return 1; + } + + in.close(); + out.close(); + + return 0; +}