Adding an option to set the digest function that everything uses. Minor refactoring: enabling potential fast digest computation of more than one digest function type.
Usage: bazel --host_jvm_args="-Dbazel.DigestFunction=SHA1" build ...
Ugliness: using a system property (a static non-final variable), because the better way to do it (a flag) would result in a much, much larger refactoring.
More ugliness: I have updated the minimal amount of tests. A lot of tests are still relying on the default value of MD5. Ideally, they need to be updated as well.
--
MOS_MIGRATED_REVID=139490836
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java
index 90aac0e..4d93804 100644
--- a/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java
+++ b/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java
@@ -16,6 +16,7 @@
import static java.nio.charset.StandardCharsets.ISO_8859_1;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.hash.Hashing;
import com.google.common.io.ByteSource;
@@ -24,6 +25,8 @@
import com.google.devtools.build.lib.vfs.Dirent.Type;
import com.google.devtools.build.lib.vfs.Path.PathFactory;
import com.google.devtools.build.lib.vfs.Path.PathFactory.TranslatedPath;
+import com.google.devtools.common.options.EnumConverter;
+import com.google.devtools.common.options.OptionsParsingException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -39,6 +42,49 @@
@ThreadSafe
public abstract class FileSystem {
+ /** Type of hash function to use for digesting files. */
+ public enum HashFunction {
+ MD5(16),
+ SHA1(20);
+
+ private final int digestSize;
+
+ HashFunction(int digestSize) {
+ this.digestSize = digestSize;
+ }
+
+ /** Converts to {@link HashFunction}. */
+ public static class Converter extends EnumConverter<HashFunction> {
+ public Converter() {
+ super(HashFunction.class, "hash function");
+ }
+ }
+
+ public boolean isValidDigest(byte[] digest) {
+ return digest != null && digest.length == digestSize;
+ }
+ }
+
+ // This is effectively final, should be changed only in unit-tests!
+ private static HashFunction DIGEST_FUNCTION;
+ static {
+ try {
+ DIGEST_FUNCTION = new HashFunction.Converter().convert(
+ System.getProperty("bazel.DigestFunction", "MD5"));
+ } catch (OptionsParsingException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @VisibleForTesting
+ public static void setDigestFunctionForTesting(HashFunction value) {
+ DIGEST_FUNCTION = value;
+ }
+
+ public static HashFunction getDigestFunction() {
+ return DIGEST_FUNCTION;
+ }
+
private enum UnixPathFactory implements PathFactory {
INSTANCE {
@Override
@@ -261,10 +307,11 @@
}
/**
- * Returns the type of digest that may be returned by {@link #getFastDigest}, or {@code null}
- * if the filesystem doesn't support them.
+ * Gets a fast digest for the given path and hash function type, or {@code null} if there
+ * isn't one available or the filesystem doesn't support them. This digest should be
+ * suitable for detecting changes to the file.
*/
- protected String getFastDigestFunctionType(Path path) {
+ protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
return null;
}
@@ -273,8 +320,43 @@
* filesystem doesn't support them. This digest should be suitable for detecting changes to the
* file.
*/
- protected byte[] getFastDigest(Path path) throws IOException {
- return null;
+ protected final byte[] getFastDigest(Path path) throws IOException {
+ return getFastDigest(path, DIGEST_FUNCTION);
+ }
+
+ /**
+ * Returns whether the given digest is a valid digest for the default digest function.
+ */
+ public boolean isValidDigest(byte[] digest) {
+ return DIGEST_FUNCTION.isValidDigest(digest);
+ }
+
+ /**
+ * Returns the digest of the file denoted by the path, following
+ * symbolic links, for the given hash digest function.
+ *
+ * @return a new byte array containing the file's digest
+ * @throws IOException if the digest could not be computed for any reason
+ */
+ protected final byte[] getDigest(final Path path, HashFunction hashFunction) throws IOException {
+ switch(hashFunction) {
+ case MD5:
+ return getMD5Digest(path);
+ case SHA1:
+ return getSHA1Digest(path);
+ default:
+ throw new IOException("Unsupported hash function: " + hashFunction);
+ }
+ }
+
+ /**
+ * Returns the digest of the file denoted by the path, following symbolic links.
+ *
+ * @return a new byte array containing the file's digest
+ * @throws IOException if the digest could not be computed for any reason
+ */
+ protected byte[] getDigest(final Path path) throws IOException {
+ return getDigest(path, DIGEST_FUNCTION);
}
/**
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Path.java b/src/main/java/com/google/devtools/build/lib/vfs/Path.java
index 6599952..e039efe 100644
--- a/src/main/java/com/google/devtools/build/lib/vfs/Path.java
+++ b/src/main/java/com/google/devtools/build/lib/vfs/Path.java
@@ -19,6 +19,7 @@
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.util.StringCanonicalizer;
+import com.google.devtools.build.lib.vfs.FileSystem.HashFunction;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -1039,19 +1040,26 @@
}
/**
- * Returns the type of digest that may be returned by {@link #getFastDigest}, or {@code null}
- * if the filesystem doesn't support them.
+ * Gets a fast digest for the given path, or {@code null} if there isn't one available. The
+ * digest should be suitable for detecting changes to the file.
*/
- public String getFastDigestFunctionType() {
- return fileSystem.getFastDigestFunctionType(this);
+ public byte[] getFastDigest() throws IOException {
+ return fileSystem.getFastDigest(this);
}
/**
* Gets a fast digest for the given path, or {@code null} if there isn't one available. The
* digest should be suitable for detecting changes to the file.
*/
- public byte[] getFastDigest() throws IOException {
- return fileSystem.getFastDigest(this);
+ public byte[] getFastDigest(HashFunction hashFunction) throws IOException {
+ return fileSystem.getFastDigest(this, hashFunction);
+ }
+
+ /**
+ * Returns whether the given digest is a valid digest for the default system digest function.
+ */
+ public boolean isValidDigest(byte[] digest) {
+ return fileSystem.isValidDigest(digest);
}
/**
@@ -1083,6 +1091,28 @@
}
/**
+ * Returns the digest of the file denoted by the current path,
+ * following symbolic links.
+ *
+ * @return a new byte array containing the file's digest
+ * @throws IOException if the digest could not be computed for any reason
+ */
+ public byte[] getDigest() throws IOException {
+ return fileSystem.getDigest(this);
+ }
+
+ /**
+ * Returns the digest of the file denoted by the current path and digest function,
+ * following symbolic links.
+ *
+ * @return a new byte array containing the file's digest
+ * @throws IOException if the digest could not be computed for any reason
+ */
+ public byte[] getDigest(HashFunction hashFunction) throws IOException {
+ return fileSystem.getDigest(this, hashFunction);
+ }
+
+ /**
* Opens the file denoted by this path, following symbolic links, for reading,
* and returns an input stream to it.
*
diff --git a/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java
index a0a0955..b246aa6 100644
--- a/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java
+++ b/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java
@@ -18,6 +18,7 @@
import com.google.devtools.build.lib.concurrent.ThreadSafety;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.util.StringTrie;
+import com.google.devtools.build.lib.vfs.FileSystem.HashFunction;
import java.io.IOException;
import java.io.InputStream;
@@ -380,15 +381,9 @@
}
@Override
- protected String getFastDigestFunctionType(Path path) {
+ protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
FileSystem delegate = getDelegate(path);
- return delegate.getFastDigestFunctionType(adjustPath(path, delegate));
- }
-
- @Override
- protected byte[] getFastDigest(Path path) throws IOException {
- FileSystem delegate = getDelegate(path);
- return delegate.getFastDigest(adjustPath(path, delegate));
+ return delegate.getFastDigest(adjustPath(path, delegate), hashFunction);
}
@Override
