Support unreadable files
--
MOS_MIGRATED_REVID=112507181
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
index 8e897fb..ae5231a 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ArtifactFunction.java
@@ -106,7 +106,7 @@
return null;
}
if (!fileValue.exists()) {
- if (allowedMissingInputs.apply(((RootedPath) fileSkyKey.argument()).getRelativePath())) {
+ if (isAllowedMissingInput(fileSkyKey)) {
return FileArtifactValue.MISSING_FILE_MARKER;
} else {
return missingInputFile(artifact, mandatory, null, env.getListener());
@@ -115,10 +115,17 @@
try {
return FileArtifactValue.create(artifact, fileValue);
} catch (IOException e) {
+ if (isAllowedMissingInput(fileSkyKey)) {
+ return FileArtifactValue.MISSING_FILE_MARKER;
+ }
throw makeMissingInputFileExn(artifact, mandatory, e, env.getListener());
}
}
+ private boolean isAllowedMissingInput(SkyKey fileSkyKey) {
+ return allowedMissingInputs.apply(((RootedPath) fileSkyKey.argument()).getRelativePath());
+ }
+
private static ArtifactValue missingInputFile(Artifact artifact, boolean mandatory,
Exception failure, EventHandler reporter) throws MissingInputFileException {
if (!mandatory) {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java
index 80b43a7..459d1ca 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java
@@ -164,11 +164,9 @@
@Nullable TimestampGranularityMonitor tsgm)
throws InconsistentFilesystemException {
Preconditions.checkState(stat.isFile(), path);
+
try {
- byte[] digest = stat.getDigest();
- if (digest == null) {
- digest = path.getFastDigest();
- }
+ byte[] digest = tryGetDigest(path, stat);
if (digest == null) {
long mtime = stat.getLastModifiedTime();
// Note that TimestampGranularityMonitor#notifyDependenceOnFileTime is a thread-safe
@@ -193,6 +191,19 @@
}
}
+ @Nullable
+ private static byte[] tryGetDigest(Path path, FileStatusWithDigest stat) throws IOException {
+ try {
+ byte[] digest = stat.getDigest();
+ return digest != null ? digest : path.getFastDigest();
+ } catch (IOException ioe) {
+ if (!path.isReadable()) {
+ return null;
+ }
+ throw ioe;
+ }
+ }
+
@Override
public Type getType() {
return Type.REGULAR_FILE;
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java
index 5ca1354..aae3254 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java
@@ -21,7 +21,7 @@
import static org.junit.Assert.assertSame;
import static org.junit.Assert.fail;
-import com.google.common.base.Predicates;
+import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -64,6 +64,8 @@
import org.junit.runners.JUnit4;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -82,6 +84,14 @@
private static final SkyKey OWNER_KEY = new SkyKey(SkyFunctions.ACTION_LOOKUP, "OWNER");
private static final ActionLookupKey ALL_OWNER = new SingletonActionLookupKey();
+ private PathFragment allowedMissingInput = null;
+ private Predicate<PathFragment> allowedMissingInputsPredicate = new Predicate<PathFragment>() {
+ @Override
+ public boolean apply(PathFragment input) {
+ return input.equals(allowedMissingInput);
+ }
+ };
+
private Set<Action> actions;
private boolean fastDigest = false;
private RecordingDifferencer differencer = new RecordingDifferencer();
@@ -102,8 +112,7 @@
ImmutableMap.<SkyFunctionName, SkyFunction>builder()
.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper))
.put(SkyFunctions.FILE, new FileFunction(pkgLocator))
- .put(SkyFunctions.ARTIFACT,
- new ArtifactFunction(Predicates.<PathFragment>alwaysFalse()))
+ .put(SkyFunctions.ARTIFACT, new ArtifactFunction(allowedMissingInputsPredicate))
.put(SkyFunctions.ACTION_EXECUTION, new SimpleActionExecutionFunction())
.put(SkyFunctions.PACKAGE,
new PackageFunction(null, null, null, null, null, null, null))
@@ -152,6 +161,52 @@
}
@Test
+ public void testMissingMandatoryAllowedMissingArtifact() throws Throwable {
+ Artifact input = createSourceArtifact("allowedMissing");
+ allowedMissingInput = input.getRootRelativePath();
+ assertThat(evaluateArtifactValue(input, /*mandatory=*/ true))
+ .isEqualTo(FileArtifactValue.MISSING_FILE_MARKER);
+ }
+
+ @Test
+ public void testUnreadableMandatoryAllowedMissingArtifact() throws Throwable {
+ Artifact input = createSourceArtifact("allowedMissing");
+ file(input.getPath(), "allowedMissing");
+ input.getPath().chmod(0);
+
+ allowedMissingInput = input.getRootRelativePath();
+ assertThat(evaluateArtifactValue(input, /*mandatory=*/ true))
+ .isEqualTo(FileArtifactValue.MISSING_FILE_MARKER);
+ }
+
+ @Test
+ public void testUnreadableInputWithFsWithAvailableDigest() throws Throwable {
+ final byte[] expectedDigest = MessageDigest.getInstance("md5").digest(
+ "someunreadablecontent".getBytes(StandardCharsets.UTF_8));
+ setupRoot(
+ new CustomInMemoryFs() {
+ @Override
+ public byte[] getMD5Digest(Path path) throws IOException {
+ return path.getBaseName().equals("unreadable")
+ ? expectedDigest
+ : super.getMD5Digest(path);
+ }
+ });
+
+ Artifact input = createSourceArtifact("unreadable");
+ Path inputPath = input.getPath();
+ file(inputPath, "dummynotused");
+ inputPath.chmod(0);
+
+ FileArtifactValue value =
+ (FileArtifactValue) evaluateArtifactValue(input, /*mandatory=*/ true);
+
+ FileStatus stat = inputPath.stat();
+ assertThat(value.getSize()).isEqualTo(stat.getSize());
+ assertThat(value.getDigest()).isEqualTo(expectedDigest);
+ }
+
+ @Test
public void testMissingMandatoryArtifact() throws Throwable {
Artifact input = createSourceArtifact("input1");
try {
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java
index 873d48d..d9cce8b 100644
--- a/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java
+++ b/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java
@@ -76,6 +76,8 @@
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -437,6 +439,44 @@
}
@Test
+ public void testUnreadableFileWithNoFastDigest() throws Exception {
+ Path p = file("unreadable");
+ p.chmod(0);
+ p.setLastModifiedTime(0L);
+
+ FileValue value = valueForPath(p);
+ assertTrue(value.exists());
+ assertThat(value.getDigest()).isNull();
+
+ p.setLastModifiedTime(10L);
+ assertThat(valueForPath(p)).isNotEqualTo(value);
+
+ p.setLastModifiedTime(0L);
+ assertThat(valueForPath(p)).isEqualTo(value);
+ }
+
+ @Test
+ public void testUnreadableFileWithFastDigest() throws Exception {
+ final byte[] expectedDigest = MessageDigest.getInstance("md5").digest(
+ "blah".getBytes(StandardCharsets.UTF_8));
+
+ createFsAndRoot(
+ new CustomInMemoryFs(manualClock) {
+ @Override
+ protected byte[] getFastDigest(Path path) {
+ return path.getBaseName().equals("unreadable") ? expectedDigest : null;
+ }
+ });
+
+ Path p = file("unreadable");
+ p.chmod(0);
+
+ FileValue value = valueForPath(p);
+ assertThat(value.exists()).isTrue();
+ assertThat(value.getDigest()).isNotNull();
+ }
+
+ @Test
public void testFileModificationModTime() throws Exception {
fastMd5 = false;
Path p = file("file");
@@ -882,6 +922,36 @@
assertThat(errorInfo.getException().getMessage()).contains("/root/a is no longer a file");
}
+ @Test
+ public void testFilesystemInconsistencies_GetFastDigestAndIsReadableFailure() throws Exception {
+ createFsAndRoot(
+ new CustomInMemoryFs(manualClock) {
+ @Override
+ protected boolean isReadable(Path path) throws IOException {
+ if (path.getBaseName().equals("unreadable")) {
+ throw new IOException("isReadable failed");
+ }
+ return super.isReadable(path);
+ }
+ });
+
+ Path p = file("unreadable");
+ p.chmod(0);
+
+ SequentialBuildDriver driver = makeDriver();
+ SkyKey skyKey = skyKey("unreadable");
+ EvaluationResult<FileValue> result =
+ driver.evaluate(
+ ImmutableList.of(skyKey), false, DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
+ assertTrue(result.hasError());
+ ErrorInfo errorInfo = result.getError(skyKey);
+ assertThat(errorInfo.getException()).isInstanceOf(InconsistentFilesystemException.class);
+ assertThat(errorInfo.getException().getMessage())
+ .contains("encountered error 'isReadable failed'");
+ assertThat(errorInfo.getException().getMessage())
+ .contains("/root/unreadable is no longer a file");
+ }
+
private void runTestSymlinkCycle(boolean ancestorCycle, boolean startInCycle) throws Exception {
symlink("a", "b");
symlink("b", "c");
