Add BLAKE3 hasher to vfs
This PR adds the Blake3Hasher and Blake3HashFunction classes to vfs and
makes them available under the flag --digest_function=BLAKE3.
This is a partial commit for #18658.
Closes #18784.
PiperOrigin-RevId: 550525978
Change-Id: Iedc0886c51755585d56b4d8f47676d3be5bbedba
diff --git a/BUILD b/BUILD
index 84c15b6..0540bb8 100644
--- a/BUILD
+++ b/BUILD
@@ -96,6 +96,7 @@
pkg_tar(
name = "bootstrap-jars",
srcs = [
+ "@blake3",
"@com_google_protobuf//:protobuf_java",
"@com_google_protobuf//:protobuf_java_util",
"@com_google_protobuf//:protobuf_javalite",
diff --git a/MODULE.bazel b/MODULE.bazel
index 4c14098..4c4d4fe 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -14,6 +14,7 @@
bazel_dep(name = "rules_pkg", version = "0.7.0")
bazel_dep(name = "stardoc", version = "0.5.3", repo_name = "io_bazel_skydoc")
bazel_dep(name = "zstd-jni", version = "1.5.2-3")
+bazel_dep(name = "blake3", version = "1.3.3")
bazel_dep(name = "zlib", version = "1.2.13")
bazel_dep(name = "rules_cc", version = "0.0.8")
bazel_dep(name = "rules_java", version = "6.2.2")
diff --git a/distdir_deps.bzl b/distdir_deps.bzl
index cf9bd19..ddfa94b 100644
--- a/distdir_deps.bzl
+++ b/distdir_deps.bzl
@@ -272,7 +272,7 @@
"package_version": "1.5.2-3",
},
"blake3": {
- "archive": "v1.3.3.zip",
+ "archive": "1.3.3.zip",
"sha256": "bb529ba133c0256df49139bd403c17835edbf60d2ecd6463549c6a5fe279364d",
"urls": [
"https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/1.3.3.zip",
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index 27ac03a..ac88542 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -96,6 +96,7 @@
"//src/main/java/com/google/devtools/build/lib/util:srcs",
"//src/main/java/com/google/devtools/build/lib/versioning:srcs",
"//src/main/java/com/google/devtools/build/lib/vfs:srcs",
+ "//src/main/java/com/google/devtools/build/lib/vfs/bazel:srcs",
"//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs:srcs",
"//src/main/java/com/google/devtools/build/lib/windows:srcs",
"//src/main/java/com/google/devtools/build/lib/worker:srcs",
@@ -443,6 +444,7 @@
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:output_service",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+ "//src/main/java/com/google/devtools/build/lib/vfs/bazel",
"//src/main/java/com/google/devtools/build/lib/windows",
"//src/main/java/com/google/devtools/build/lib/worker:worker_metric",
"//src/main/java/com/google/devtools/build/skyframe",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD
index 087438d..4097239 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD
@@ -157,6 +157,7 @@
"//src/main/java/com/google/devtools/build/lib/util:os",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+ "//src/main/java/com/google/devtools/build/lib/vfs/bazel",
"//src/main/java/com/google/devtools/build/lib/windows",
"//src/main/java/com/google/devtools/common/options",
"//src/main/protobuf:failure_details_java_proto",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelFileSystemModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelFileSystemModule.java
index 1027f3a..f5af64e 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelFileSystemModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelFileSystemModule.java
@@ -31,6 +31,7 @@
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.JavaIoFileSystem;
import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.bazel.BazelHashFunctions;
import com.google.devtools.build.lib.windows.WindowsFileSystem;
import com.google.devtools.common.options.OptionsParsingException;
import com.google.devtools.common.options.OptionsParsingResult;
@@ -44,6 +45,10 @@
* com.google.devtools.build.lib.vfs.FileSystem} class use {@code SHA256} by default.
*/
public class BazelFileSystemModule extends BlazeModule {
+ static {
+ BazelHashFunctions.ensureRegistered();
+ }
+
@Override
public ModuleFileSystem getFileSystem(
OptionsParsingResult startupOptions, PathFragment realExecRootBase)
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BUILD
new file mode 100644
index 0000000..fde90bc
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BUILD
@@ -0,0 +1,28 @@
+load("@rules_java//java:defs.bzl", "java_library")
+
+package(
+ default_applicable_licenses = ["//:license"],
+ default_visibility = ["//src:__subpackages__"],
+)
+
+filegroup(
+ name = "srcs",
+ srcs = glob(["**"]),
+ visibility = ["//src:__subpackages__"],
+)
+
+java_library(
+ name = "bazel",
+ srcs = glob(
+ [
+ "*.java",
+ ],
+ ),
+ deps = [
+ "//src/main/java/com/google/devtools/build/lib/jni",
+ "//src/main/java/com/google/devtools/build/lib/vfs",
+ "//third_party:error_prone_annotations",
+ "//third_party:guava",
+ "//third_party:jsr305",
+ ],
+)
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctions.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctions.java
new file mode 100644
index 0000000..c917441
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctions.java
@@ -0,0 +1,45 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.bazel;
+
+import com.google.devtools.build.lib.jni.JniLoader;
+import com.google.devtools.build.lib.vfs.DigestHashFunction;
+import java.security.Security;
+import javax.annotation.Nullable;
+
+/** Bazel specific {@link DigestHashFunction}s. */
+public final class BazelHashFunctions {
+ @Nullable public static final DigestHashFunction BLAKE3;
+
+ static {
+ DigestHashFunction hashFunction = null;
+
+ if (JniLoader.isJniAvailable()) {
+ try {
+ Security.addProvider(new Blake3Provider());
+ hashFunction = DigestHashFunction.register(new Blake3HashFunction(), "BLAKE3");
+ } catch (UnsatisfiedLinkError ignored) {
+ // This can happen if bazel was compiled manually (with compile.sh),
+ // on windows. In that case jni is available, but missing the blake3
+ // symbols necessary to register the hasher.
+ }
+ }
+
+ BLAKE3 = hashFunction;
+ }
+
+ public static void ensureRegistered() {}
+
+ private BazelHashFunctions() {}
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3HashFunction.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3HashFunction.java
new file mode 100644
index 0000000..0b203ba
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3HashFunction.java
@@ -0,0 +1,88 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.bazel;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkPositionIndexes;
+
+import com.google.common.hash.Funnel;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+
+/** A {@link HashFunction} for BLAKE3. */
+public final class Blake3HashFunction implements HashFunction {
+ @Override
+ public int bits() {
+ return 256;
+ }
+
+ @Override
+ public Hasher newHasher() {
+ return new Blake3Hasher(new Blake3MessageDigest());
+ }
+
+ @Override
+ public Hasher newHasher(int expectedInputSize) {
+ checkArgument(
+ expectedInputSize >= 0, "expectedInputSize must be >= 0 but was %s", expectedInputSize);
+ return newHasher();
+ }
+
+ /* The following methods implement the {HashFunction} interface. */
+
+ @Override
+ public <T> HashCode hashObject(T instance, Funnel<? super T> funnel) {
+ return newHasher().putObject(instance, funnel).hash();
+ }
+
+ @Override
+ public HashCode hashUnencodedChars(CharSequence input) {
+ int len = input.length();
+ return newHasher(len * 2).putUnencodedChars(input).hash();
+ }
+
+ @Override
+ public HashCode hashString(CharSequence input, Charset charset) {
+ return newHasher().putString(input, charset).hash();
+ }
+
+ @Override
+ public HashCode hashInt(int input) {
+ return newHasher(4).putInt(input).hash();
+ }
+
+ @Override
+ public HashCode hashLong(long input) {
+ return newHasher(8).putLong(input).hash();
+ }
+
+ @Override
+ public HashCode hashBytes(byte[] input) {
+ return hashBytes(input, 0, input.length);
+ }
+
+ @Override
+ public HashCode hashBytes(byte[] input, int off, int len) {
+ checkPositionIndexes(off, off + len, input.length);
+ return newHasher(len).putBytes(input, off, len).hash();
+ }
+
+ @Override
+ public HashCode hashBytes(ByteBuffer input) {
+ return newHasher(input.remaining()).putBytes(input).hash();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Hasher.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Hasher.java
new file mode 100644
index 0000000..180d62b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Hasher.java
@@ -0,0 +1,146 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.bazel;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.hash.Funnel;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hasher;
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+
+/** A {@link Hasher} for BLAKE3. */
+public final class Blake3Hasher implements Hasher {
+ private final Blake3MessageDigest messageDigest;
+ private boolean isDone = false;
+
+ public Blake3Hasher(Blake3MessageDigest blake3MessageDigest) {
+ messageDigest = blake3MessageDigest;
+ }
+
+ /* The following methods implement the {Hasher} interface. */
+
+ @Override
+ @CanIgnoreReturnValue
+ public Hasher putBytes(ByteBuffer b) {
+ messageDigest.engineUpdate(b);
+ return this;
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public Hasher putBytes(byte[] bytes, int off, int len) {
+ messageDigest.engineUpdate(bytes, off, len);
+ return this;
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public Hasher putBytes(byte[] bytes) {
+ messageDigest.engineUpdate(bytes, 0, bytes.length);
+ return this;
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public Hasher putByte(byte b) {
+ messageDigest.engineUpdate(b);
+ return this;
+ }
+
+ @Override
+ public HashCode hash() {
+ checkState(!isDone);
+ isDone = true;
+
+ return HashCode.fromBytes(messageDigest.engineDigest());
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public final Hasher putBoolean(boolean b) {
+ return putByte(b ? (byte) 1 : (byte) 0);
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public final Hasher putDouble(double d) {
+ return putLong(Double.doubleToRawLongBits(d));
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public final Hasher putFloat(float f) {
+ return putInt(Float.floatToRawIntBits(f));
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public Hasher putUnencodedChars(CharSequence charSequence) {
+ for (int i = 0, len = charSequence.length(); i < len; i++) {
+ putChar(charSequence.charAt(i));
+ }
+ return this;
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public Hasher putString(CharSequence charSequence, Charset charset) {
+ return putBytes(charSequence.toString().getBytes(charset));
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public Hasher putShort(short s) {
+ putByte((byte) s);
+ putByte((byte) (s >>> 8));
+ return this;
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public Hasher putInt(int i) {
+ putByte((byte) i);
+ putByte((byte) (i >>> 8));
+ putByte((byte) (i >>> 16));
+ putByte((byte) (i >>> 24));
+ return this;
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public Hasher putLong(long l) {
+ for (int i = 0; i < 64; i += 8) {
+ putByte((byte) (l >>> i));
+ }
+ return this;
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public Hasher putChar(char c) {
+ putByte((byte) c);
+ putByte((byte) (c >>> 8));
+ return this;
+ }
+
+ @Override
+ @CanIgnoreReturnValue
+ public <T> Hasher putObject(T instance, Funnel<? super T> funnel) {
+ funnel.funnel(instance, this);
+ return this;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3MessageDigest.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3MessageDigest.java
new file mode 100644
index 0000000..50d2ace
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3MessageDigest.java
@@ -0,0 +1,139 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.bazel;
+
+import static java.lang.Math.min;
+
+import com.google.devtools.build.lib.jni.JniLoader;
+import java.nio.ByteBuffer;
+import java.security.DigestException;
+import java.security.MessageDigest;
+
+/** A {@link MessageDigest} for BLAKE3. */
+public final class Blake3MessageDigest extends MessageDigest {
+ // These constants match the native definitions in:
+ // https://github.com/BLAKE3-team/BLAKE3/blob/master/c/blake3.h
+ public static final int KEY_LEN = 32;
+ public static final int OUT_LEN = 32;
+
+ static {
+ JniLoader.loadJni();
+ }
+
+ private static final int STATE_SIZE = hasher_size();
+ private static final byte[] INITIAL_STATE = new byte[STATE_SIZE];
+
+ static {
+ initialize_hasher(INITIAL_STATE);
+ }
+
+ // To reduce the number of calls made via JNI, buffer up to this many bytes
+ // before updating the hasher.
+ public static final int ONESHOT_THRESHOLD = 8 * 1024;
+
+ private final ByteBuffer buffer = ByteBuffer.allocate(ONESHOT_THRESHOLD);
+ private final byte[] hasher = new byte[STATE_SIZE];
+
+ public Blake3MessageDigest() {
+ super("BLAKE3");
+ System.arraycopy(INITIAL_STATE, 0, hasher, 0, STATE_SIZE);
+ }
+
+ private void flush() {
+ if (buffer.position() > 0) {
+ blake3_hasher_update(hasher, buffer.array(), buffer.position());
+ buffer.clear();
+ }
+ }
+
+ @Override
+ public void engineUpdate(byte[] data, int offset, int length) {
+ while (length > 0) {
+ int numToCopy = min(length, buffer.remaining());
+ buffer.put(data, offset, numToCopy);
+ length -= numToCopy;
+ offset += numToCopy;
+
+ if (buffer.remaining() == 0) {
+ flush();
+ }
+ }
+ }
+
+ @Override
+ public void engineUpdate(byte b) {
+ if (buffer.remaining() == 0) {
+ flush();
+ }
+ buffer.put(b);
+ }
+
+ @Override
+ public void engineUpdate(ByteBuffer input) {
+ super.engineUpdate(input);
+ }
+
+ private byte[] getOutput(int outputLength) {
+ flush();
+
+ byte[] retByteArray = new byte[outputLength];
+ blake3_hasher_finalize(hasher, retByteArray, outputLength);
+
+ engineReset();
+ return retByteArray;
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ throw new CloneNotSupportedException();
+ }
+
+ @Override
+ public void engineReset() {
+ buffer.clear();
+ System.arraycopy(INITIAL_STATE, 0, hasher, 0, STATE_SIZE);
+ }
+
+ @Override
+ public int engineGetDigestLength() {
+ return OUT_LEN;
+ }
+
+ @Override
+ public byte[] engineDigest() {
+ return getOutput(OUT_LEN);
+ }
+
+ @Override
+ public int engineDigest(byte[] buf, int off, int len) throws DigestException {
+ if (len < OUT_LEN) {
+ throw new DigestException("partial digests not returned");
+ }
+ if (buf.length - off < OUT_LEN) {
+ throw new DigestException("insufficient space in the output buffer to store the digest");
+ }
+
+ byte[] digestBytes = getOutput(OUT_LEN);
+ System.arraycopy(digestBytes, 0, buf, off, digestBytes.length);
+ return digestBytes.length;
+ }
+
+ public static final native int hasher_size();
+
+ public static final native void initialize_hasher(byte[] hasher);
+
+ public static final native void blake3_hasher_update(byte[] hasher, byte[] input, int inputLen);
+
+ public static final native void blake3_hasher_finalize(byte[] hasher, byte[] out, int outLen);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Provider.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Provider.java
new file mode 100644
index 0000000..0c86814
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Provider.java
@@ -0,0 +1,24 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.vfs.bazel;
+
+import java.security.Provider;
+
+/** A {@link Provider} for BLAKE3. */
+public final class Blake3Provider extends Provider {
+ public Blake3Provider() {
+ super("BLAKE3Provider", "1.0", "A BLAKE3 digest provider");
+ put("MessageDigest.BLAKE3", Blake3MessageDigest.class.getName());
+ }
+}
diff --git a/src/main/native/BUILD b/src/main/native/BUILD
index 2601d32..c011aef 100644
--- a/src/main/native/BUILD
+++ b/src/main/native/BUILD
@@ -54,6 +54,21 @@
includes = ["."], # For jni headers.
)
+cc_library(
+ name = "blake3_jni",
+ srcs = [
+ "blake3_jni.cc",
+ ":jni.h",
+ ":jni_md.h",
+ ],
+ includes = ["."], # For jni headers.
+ visibility = ["//src/main/native:__subpackages__"],
+ deps = [
+ "@blake3",
+ ],
+ alwayslink = 1,
+)
+
cc_binary(
name = "libunix_jni.so",
srcs = [
@@ -80,6 +95,7 @@
linkshared = 1,
visibility = ["//src/main/java/com/google/devtools/build/lib/jni:__pkg__"],
deps = [
+ ":blake3_jni",
":latin1_jni_path",
"//src/main/cpp/util:logging",
"//src/main/cpp/util:md5",
diff --git a/src/main/native/blake3_jni.cc b/src/main/native/blake3_jni.cc
new file mode 100644
index 0000000..5620188
--- /dev/null
+++ b/src/main/native/blake3_jni.cc
@@ -0,0 +1,73 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <jni.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "c/blake3.h"
+
+namespace blaze_jni {
+
+jbyte *get_byte_array(JNIEnv *env, jbyteArray java_array) {
+ return (jbyte *)env->GetPrimitiveArrayCritical(java_array, nullptr);
+}
+
+void release_byte_array(JNIEnv *env, jbyteArray array, jbyte *addr) {
+ env->ReleasePrimitiveArrayCritical(array, addr, 0);
+}
+
+extern "C" JNIEXPORT int JNICALL
+Java_com_google_devtools_build_lib_vfs_bazel_Blake3MessageDigest_hasher_1size(
+ JNIEnv *env, jobject obj) {
+ return (int)sizeof(blake3_hasher);
+}
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_vfs_bazel_Blake3MessageDigest_initialize_1hasher(
+ JNIEnv *env, jobject obj, jbyteArray jhasher) {
+ blake3_hasher *hasher = (blake3_hasher *)get_byte_array(env, jhasher);
+ if (hasher) {
+ blake3_hasher_init(hasher);
+ release_byte_array(env, jhasher, (jbyte *)hasher);
+ }
+}
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_vfs_bazel_Blake3MessageDigest_blake3_1hasher_1update(
+ JNIEnv *env, jobject obj, jbyteArray jhasher, jbyteArray input,
+ jint input_len) {
+ blake3_hasher *hasher = (blake3_hasher *)get_byte_array(env, jhasher);
+ if (hasher) {
+ jbyte *input_addr = get_byte_array(env, input);
+ blake3_hasher_update(hasher, input_addr, input_len);
+ release_byte_array(env, input, input_addr);
+ release_byte_array(env, jhasher, (jbyte *)hasher);
+ }
+}
+
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_devtools_build_lib_vfs_bazel_Blake3MessageDigest_blake3_1hasher_1finalize(
+ JNIEnv *env, jobject obj, jbyteArray jhasher, jbyteArray out,
+ jint out_len) {
+ blake3_hasher *hasher = (blake3_hasher *)get_byte_array(env, jhasher);
+ if (hasher) {
+ jbyte *out_addr = get_byte_array(env, out);
+ blake3_hasher_finalize(hasher, (uint8_t *)out_addr, out_len);
+ release_byte_array(env, out, out_addr);
+ release_byte_array(env, jhasher, (jbyte *)hasher);
+ }
+}
+
+} // namespace blaze_jni
diff --git a/src/main/native/windows/BUILD b/src/main/native/windows/BUILD
index 73eaeed..b595e75 100644
--- a/src/main/native/windows/BUILD
+++ b/src/main/native/windows/BUILD
@@ -76,6 +76,7 @@
deps = [
":lib-file",
":lib-process",
+ "//src/main/native:blake3_jni",
],
)
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/BUILD b/src/test/java/com/google/devtools/build/lib/vfs/BUILD
index 8b9e008..d4856a8 100644
--- a/src/test/java/com/google/devtools/build/lib/vfs/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/vfs/BUILD
@@ -10,6 +10,7 @@
name = "srcs",
testonly = 0,
srcs = glob(["**"]) + [
+ "//src/test/java/com/google/devtools/build/lib/vfs/bazel:srcs",
"//src/test/java/com/google/devtools/build/lib/vfs/util:srcs",
],
visibility = ["//src:__subpackages__"],
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD
new file mode 100644
index 0000000..f1d34ce
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD
@@ -0,0 +1,42 @@
+load("@rules_java//java:defs.bzl", "java_library", "java_test")
+
+package(
+ default_applicable_licenses = ["//:license"],
+ default_testonly = 1,
+ default_visibility = ["//src:__subpackages__"],
+)
+
+filegroup(
+ name = "srcs",
+ testonly = 0,
+ srcs = glob(["**"]),
+ visibility = ["//src:__subpackages__"],
+)
+
+java_library(
+ name = "BazelTests_lib",
+ srcs = glob(
+ [
+ "*.java",
+ ],
+ ),
+ deps = [
+ "//src/main/java/com/google/devtools/build/lib/vfs/bazel",
+ "//src/test/java/com/google/devtools/build/lib/testutil",
+ "//third_party:guava",
+ "//third_party:guava-testlib",
+ "//third_party:junit4",
+ "//third_party:truth",
+ "//third_party/protobuf:protobuf_java",
+ ],
+)
+
+java_test(
+ name = "BazelTests",
+ size = "small",
+ test_class = "com.google.devtools.build.lib.AllTests",
+ runtime_deps = [
+ ":BazelTests_lib",
+ "//src/test/java/com/google/devtools/build/lib:test_runner",
+ ],
+)
diff --git a/src/test/java/com/google/devtools/build/lib/vfs/bazel/Blake3HasherTest.java b/src/test/java/com/google/devtools/build/lib/vfs/bazel/Blake3HasherTest.java
new file mode 100644
index 0000000..9fa0cb3
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/vfs/bazel/Blake3HasherTest.java
@@ -0,0 +1,48 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.vfs.bazel;
+
+import static org.junit.Assert.assertEquals;
+
+import java.nio.charset.StandardCharsets;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link Blake3MessageDigest}. */
+@RunWith(JUnit4.class)
+public class Blake3HasherTest {
+ @Test
+ public void emptyHash() {
+ Blake3Hasher h = new Blake3Hasher(new Blake3MessageDigest());
+
+ byte[] data = new byte[0];
+ h.putBytes(data);
+
+ assertEquals(
+ "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", h.hash().toString());
+ }
+
+ @Test
+ public void helloWorld() {
+ Blake3Hasher h = new Blake3Hasher(new Blake3MessageDigest());
+
+ byte[] data = "hello world".getBytes(StandardCharsets.US_ASCII);
+ h.putBytes(data);
+
+ assertEquals(
+ "d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24", h.hash().toString());
+ }
+}
diff --git a/third_party/blake3/blake3.BUILD b/third_party/blake3/blake3.BUILD
index 6e96f05..867e33f 100644
--- a/third_party/blake3/blake3.BUILD
+++ b/third_party/blake3/blake3.BUILD
@@ -30,7 +30,9 @@
] + select({
"@bazel_tools//src/conditions:linux_x86_64": [
"c/blake3_avx2_x86-64_unix.S",
- "c/blake3_avx512_x86-64_unix.S",
+ # Disable to appease bazel-ci which uses ubuntu-18 (EOL) and GCC 7
+ # lacking the headers to compile AVX512.
+ # "c/blake3_avx512_x86-64_unix.S",
"c/blake3_sse2_x86-64_unix.S",
"c/blake3_sse41_x86-64_unix.S",
],
@@ -50,16 +52,21 @@
"c/blake3_impl.h",
],
copts = select({
- "@bazel_tools//src/conditions:linux_x86_64": [],
+ "@bazel_tools//src/conditions:linux_x86_64": [
+ # Disable to appease bazel-ci which uses ubuntu-18 (EOL) and GCC 7
+ # lacking the headers to compile AVX512.
+ "-DBLAKE3_NO_AVX512",
+ ],
"@bazel_tools//src/conditions:windows_x64": [],
"@bazel_tools//src/conditions:darwin_arm64": [
"-DBLAKE3_USE_NEON=1",
],
"//conditions:default": [
- "-DBLAKE3_NO_SSE2",
- "-DBLAKE3_NO_SSE41",
"-DBLAKE3_NO_AVX2",
"-DBLAKE3_NO_AVX512",
+ "-DBLAKE3_NO_NEON",
+ "-DBLAKE3_NO_SSE2",
+ "-DBLAKE3_NO_SSE41",
],
}),
includes = ["."],
