blob: 7626cbd78bc48deec6873671c06890462fa1b425 [file] [log] [blame]
// Copyright 2015 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.hash.HashCode;
import com.google.common.util.concurrent.Runnables;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionLookupData;
import com.google.devtools.build.lib.actions.ActionLookupKey;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.ArchivedTreeArtifact;
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.ArtifactRoot;
import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
import com.google.devtools.build.lib.actions.FileStateValue;
import com.google.devtools.build.lib.actions.FileValue;
import com.google.devtools.build.lib.actions.RemoteArtifactChecker;
import com.google.devtools.build.lib.actions.ThreadStateReceiver;
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
import com.google.devtools.build.lib.actions.util.TestAction;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.ServerDirectories;
import com.google.devtools.build.lib.clock.BlazeClock;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.events.NullEventHandler;
import com.google.devtools.build.lib.io.FileSymlinkCycleUniquenessFunction;
import com.google.devtools.build.lib.io.FileSymlinkInfiniteExpansionUniquenessFunction;
import com.google.devtools.build.lib.packages.WorkspaceFileValue;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction;
import com.google.devtools.build.lib.skyframe.FilesystemValueChecker.ModifiedOutputsReceiver;
import com.google.devtools.build.lib.skyframe.PackageFunction.GlobbingStrategy;
import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy;
import com.google.devtools.build.lib.testutil.ManualClock;
import com.google.devtools.build.lib.testutil.TestConstants;
import com.google.devtools.build.lib.testutil.TestPackageFactoryBuilderFactory;
import com.google.devtools.build.lib.testutil.TestRuleClassProvider;
import com.google.devtools.build.lib.testutil.TestUtils;
import com.google.devtools.build.lib.testutil.TimestampGranularityUtils;
import com.google.devtools.build.lib.util.io.OutErr;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.BatchStat;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.FileStateKey;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileStatusWithDigest;
import com.google.devtools.build.lib.vfs.FileStatusWithDigestAdapter;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.ModifiedFileSet;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.Symlinks;
import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import com.google.devtools.build.skyframe.Differencer.Diff;
import com.google.devtools.build.skyframe.Differencer.DiffWithDelta.Delta;
import com.google.devtools.build.skyframe.EvaluationContext;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
import com.google.devtools.build.skyframe.MemoizingEvaluator;
import com.google.devtools.build.skyframe.RecordingDifferencer;
import com.google.devtools.build.skyframe.SequencedRecordingDifferencer;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.testing.junit.testparameterinjector.TestParameter;
import com.google.testing.junit.testparameterinjector.TestParameterInjector;
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
/** Tests for {@link FilesystemValueChecker}. */
@RunWith(TestParameterInjector.class)
public final class FilesystemValueCheckerTest {
private static final RemoteArtifactChecker CHECK_TTL =
(file, metadata) -> metadata.isAlive(Instant.now());
private static final int FSVC_THREADS_FOR_TEST = 200;
private static final ActionLookupKey ACTION_LOOKUP_KEY =
new ActionLookupKey() {
@Override
public SkyFunctionName functionName() {
return SkyFunctionName.FOR_TESTING;
}
@Nullable
@Override
public Label getLabel() {
return null;
}
@Nullable
@Override
public BuildConfigurationKey getConfigurationKey() {
return null;
}
};
private static final ActionLookupData ACTION_LOOKUP_DATA = actionLookupData(0);
private static final EvaluationContext EVALUATION_OPTIONS =
EvaluationContext.newBuilder()
.setKeepGoing(false)
.setParallelism(SkyframeExecutor.DEFAULT_THREAD_COUNT)
.setEventHandler(NullEventHandler.INSTANCE)
.build();
private final MockFileSystem fs = new MockFileSystem();
private RecordingDifferencer differencer;
private MemoizingEvaluator evaluator;
private Path pkgRoot;
@TestParameter private BatchStatMode batchStat;
private final ModifiedOutputsReceiver mockModifiedOutputsReceiver =
mock(ModifiedOutputsReceiver.class);
private final ArgumentCaptor<Artifact> modifiedOutputsCaptor =
ArgumentCaptor.forClass(Artifact.class);
private SpecialArtifact createTreeArtifact(String relPath) throws IOException {
String outSegment = "bin";
Path outputDir = fs.getPath("/" + outSegment);
Path outputPath = outputDir.getRelative(relPath);
outputDir.createDirectory();
ArtifactRoot derivedRoot =
ArtifactRoot.asDerivedRoot(fs.getPath("/"), RootType.Output, outSegment);
return ActionsTestUtil.createTreeArtifactWithGeneratingAction(
derivedRoot,
derivedRoot.getExecPath().getRelative(derivedRoot.getRoot().relativize(outputPath)));
}
private static Delta actionValueWithTreeArtifacts(List<TreeFileArtifact> contents)
throws IOException {
return Delta.justNew(actionValueWithTreeArtifacts(contents, ImmutableList.of()));
}
private static ActionExecutionValue actionValueWithTreeArtifacts(
Iterable<TreeFileArtifact> contents, Iterable<ArchivedTreeArtifact> archivedTreeArtifacts)
throws IOException {
TreeArtifactValue.MultiBuilder treeArtifacts = TreeArtifactValue.newMultiBuilder();
for (TreeFileArtifact output : contents) {
treeArtifacts.putChild(output, createMetadataFromFileSystem(output));
}
for (ArchivedTreeArtifact archivedTreeArtifact : archivedTreeArtifacts) {
treeArtifacts.setArchivedRepresentation(
archivedTreeArtifact, createMetadataFromFileSystem(archivedTreeArtifact));
}
Map<Artifact, TreeArtifactValue> treeArtifactData = new HashMap<>();
treeArtifacts.injectTo(treeArtifactData::put);
return ActionsTestUtil.createActionExecutionValue(
/* artifactData= */ ImmutableMap.of(), ImmutableMap.copyOf(treeArtifactData));
}
private static FileArtifactValue createMetadataFromFileSystem(Artifact artifact)
throws IOException {
Path path = artifact.getPath();
FileArtifactValue noDigest =
ActionMetadataHandler.fileArtifactValueFromArtifact(
artifact,
FileStatusWithDigestAdapter.maybeAdapt(path.statIfFound(Symlinks.NOFOLLOW)),
SyscallCache.NO_CACHE,
null);
return FileArtifactValue.createFromInjectedDigest(noDigest, path.getDigest());
}
private void writeFile(Path path, String... lines) throws IOException {
// Make sure we advance the clock to detect modifications which do not change the size, which
// rely on ctime.
fs.advanceClockMillis(1);
FileSystemUtils.writeIsoLatin1(path, lines);
}
private static final class MockFileSystem extends InMemoryFileSystem {
boolean statThrowsRuntimeException;
boolean readlinkThrowsIoException;
MockFileSystem() {
this(new ManualClock());
}
private MockFileSystem(ManualClock clock) {
super(clock, DigestHashFunction.SHA256);
}
@Override
public FileStatus statIfFound(PathFragment path, boolean followSymlinks) throws IOException {
if (statThrowsRuntimeException) {
throw new RuntimeException("bork");
}
return super.statIfFound(path, followSymlinks);
}
@Override
public PathFragment readSymbolicLink(PathFragment path) throws IOException {
if (readlinkThrowsIoException) {
throw new IOException("readlink failed");
}
return super.readSymbolicLink(path);
}
void advanceClockMillis(int millis) {
((ManualClock) clock).advanceMillis(millis);
}
}
private enum BatchStatMode {
DISABLED {
@Nullable
@Override
BatchStat getBatchStat(FileSystem fileSystem) {
return null;
}
},
ENABLED {
@Override
BatchStat getBatchStat(FileSystem fileSystem) {
return (useDigest, includeLinks, paths) -> {
List<FileStatusWithDigest> stats = new ArrayList<>();
for (PathFragment pathFrag : paths) {
stats.add(
FileStatusWithDigestAdapter.maybeAdapt(
fileSystem.getPath("/").getRelative(pathFrag).statIfFound(Symlinks.NOFOLLOW)));
}
return stats;
};
}
};
@Nullable
abstract BatchStat getBatchStat(FileSystem fileSystem);
}
@Before
public void setUp() throws Exception {
ImmutableMap.Builder<SkyFunctionName, SkyFunction> skyFunctions = ImmutableMap.builder();
pkgRoot = fs.getPath("/testroot");
pkgRoot.createDirectoryAndParents();
FileSystemUtils.createEmptyFile(pkgRoot.getRelative("WORKSPACE"));
AtomicReference<PathPackageLocator> pkgLocator =
new AtomicReference<>(
new PathPackageLocator(
fs.getPath("/output_base"),
ImmutableList.of(Root.fromPath(pkgRoot)),
BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY));
BlazeDirectories directories =
new BlazeDirectories(
new ServerDirectories(pkgRoot, pkgRoot, pkgRoot),
pkgRoot,
/* defaultSystemJavabase= */ null,
TestConstants.PRODUCT_NAME);
ExternalFilesHelper externalFilesHelper =
ExternalFilesHelper.createForTesting(
pkgLocator,
ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS,
directories);
skyFunctions.put(
FileStateKey.FILE_STATE,
new FileStateFunction(
Suppliers.ofInstance(new TimestampGranularityMonitor(BlazeClock.instance())),
SyscallCache.NO_CACHE,
externalFilesHelper));
skyFunctions.put(FileValue.FILE, new FileFunction(pkgLocator, directories));
skyFunctions.put(
FileSymlinkCycleUniquenessFunction.NAME, new FileSymlinkCycleUniquenessFunction());
skyFunctions.put(
FileSymlinkInfiniteExpansionUniquenessFunction.NAME,
new FileSymlinkInfiniteExpansionUniquenessFunction());
skyFunctions.put(
SkyFunctions.PACKAGE,
new PackageFunction(
null,
null,
null,
null,
null,
/* packageProgress= */ null,
PackageFunction.ActionOnIOExceptionReadingBuildFile.UseOriginalIOException.INSTANCE,
GlobbingStrategy.SKYFRAME_HYBRID,
k -> ThreadStateReceiver.NULL_INSTANCE));
skyFunctions.put(
SkyFunctions.PACKAGE_LOOKUP,
new PackageLookupFunction(
new AtomicReference<>(ImmutableSet.of()),
CrossRepositoryLabelViolationStrategy.ERROR,
BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY,
BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER));
skyFunctions.put(
WorkspaceFileValue.WORKSPACE_FILE,
new WorkspaceFileFunction(
TestRuleClassProvider.getRuleClassProvider(),
TestPackageFactoryBuilderFactory.getInstance()
.builder(directories)
.build(TestRuleClassProvider.getRuleClassProvider(), fs),
directories,
/* bzlLoadFunctionForInlining= */ null));
skyFunctions.put(
SkyFunctions.EXTERNAL_PACKAGE,
new ExternalPackageFunction(BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER));
differencer = new SequencedRecordingDifferencer();
evaluator = new InMemoryMemoizingEvaluator(skyFunctions.buildOrThrow(), differencer);
PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID());
PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get());
}
@Before
public void setupModifiedOutputReceiverMock() {
doNothing()
.when(mockModifiedOutputsReceiver)
.reportModifiedOutputFile(anyLong(), modifiedOutputsCaptor.capture());
}
public static ImmutableList<Object[]> batchStatModes() {
return Arrays.stream(BatchStatMode.values())
.map(mode -> new BatchStatMode[] {mode})
.collect(toImmutableList());
}
@Test
public void getDirtyActionValues_unchangedEmptyTreeArtifactWithArchivedFile_noDirtyKeys()
throws Exception {
SpecialArtifact treeArtifact = createTreeArtifact("dir");
treeArtifact.getPath().createDirectoryAndParents();
ActionExecutionValue actionExecutionValue =
actionValueWithTreeArtifacts(
ImmutableList.of(),
ImmutableList.of(createArchivedTreeArtifactWithContent(treeArtifact)));
assertThat(getDirtyActionValues(actionExecutionValue)).isEmpty();
}
@Test
public void getDirtyActionValues_unchangedTreeArtifactWithArchivedFile_noDirtyKeys()
throws Exception {
SpecialArtifact treeArtifact = createTreeArtifact("dir");
ActionExecutionValue actionExecutionValue =
actionValueWithTreeArtifacts(
ImmutableList.of(
createTreeFileArtifactWithContent(treeArtifact, "file1", "content"),
createTreeFileArtifactWithContent(treeArtifact, "file2", "content2")),
ImmutableList.of(createArchivedTreeArtifactWithContent(treeArtifact)));
assertThat(getDirtyActionValues(actionExecutionValue)).isEmpty();
}
@Test
public void getDirtyActionValues_editedArchivedFileForEmptyTreeArtifact_reportsChange()
throws Exception {
SpecialArtifact treeArtifact = createTreeArtifact("dir");
treeArtifact.getPath().createDirectoryAndParents();
ArchivedTreeArtifact archivedTreeArtifact =
createArchivedTreeArtifactWithContent(treeArtifact, "old content");
ActionExecutionValue actionExecutionValue =
actionValueWithTreeArtifacts(ImmutableList.of(), ImmutableList.of(archivedTreeArtifact));
writeFile(archivedTreeArtifact.getPath(), "new content");
assertThat(getDirtyActionValues(actionExecutionValue)).containsExactly(ACTION_LOOKUP_DATA);
}
@Test
public void getDirtyActionValues_editedArchivedFileForTreeArtifact_reportsChange()
throws Exception {
SpecialArtifact treeArtifact = createTreeArtifact("dir");
ArchivedTreeArtifact archivedTreeArtifact =
createArchivedTreeArtifactWithContent(treeArtifact, "old content");
ActionExecutionValue actionExecutionValue =
actionValueWithTreeArtifacts(
ImmutableList.of(
createTreeFileArtifactWithContent(
treeArtifact, /* parentRelativePath= */ "file1", "content"),
createTreeFileArtifactWithContent(
treeArtifact, /* parentRelativePath= */ "file2", "content2")),
ImmutableList.of(archivedTreeArtifact));
writeFile(archivedTreeArtifact.getPath(), "new content");
assertThat(getDirtyActionValues(actionExecutionValue)).containsExactly(ACTION_LOOKUP_DATA);
}
@Test
public void getDirtyActionValues_deletedArchivedFileForTreeArtifact_reportsChange()
throws Exception {
SpecialArtifact treeArtifact = createTreeArtifact("dir");
ArchivedTreeArtifact archivedTreeArtifact = createArchivedTreeArtifactWithContent(treeArtifact);
ActionExecutionValue actionExecutionValue =
actionValueWithTreeArtifacts(
ImmutableList.of(
createTreeFileArtifactWithContent(
treeArtifact, /* parentRelativePath= */ "file1", "content"),
createTreeFileArtifactWithContent(
treeArtifact, /* parentRelativePath= */ "file2", "content2")),
ImmutableList.of(archivedTreeArtifact));
archivedTreeArtifact.getPath().delete();
assertThat(getDirtyActionValues(actionExecutionValue)).containsExactly(ACTION_LOOKUP_DATA);
}
@Test
public void getDirtyActionValues_deletedArchivedFileForEmptyTreeArtifact_reportsChange()
throws Exception {
SpecialArtifact treeArtifact = createTreeArtifact("dir");
ArchivedTreeArtifact archivedTreeArtifact = createArchivedTreeArtifactWithContent(treeArtifact);
ActionExecutionValue actionExecutionValue =
actionValueWithTreeArtifacts(ImmutableList.of(), ImmutableList.of(archivedTreeArtifact));
archivedTreeArtifact.getPath().delete();
assertThat(getDirtyActionValues(actionExecutionValue)).containsExactly(ACTION_LOOKUP_DATA);
}
@Test
public void getDirtyActionValues_editedFileForTreeArtifactWithArchivedFile_reportsChange()
throws Exception {
SpecialArtifact treeArtifact = createTreeArtifact("dir");
TreeFileArtifact child1 =
createTreeFileArtifactWithContent(
treeArtifact, /* parentRelativePath= */ "file1", "old content");
ActionExecutionValue actionExecutionValue =
actionValueWithTreeArtifacts(
ImmutableList.of(
child1,
createTreeFileArtifactWithContent(
treeArtifact, /* parentRelativePath= */ "file2", "content2")),
ImmutableList.of(createArchivedTreeArtifactWithContent(treeArtifact)));
writeFile(child1.getPath(), "new content");
assertThat(getDirtyActionValues(actionExecutionValue)).containsExactly(ACTION_LOOKUP_DATA);
}
@Test
public void getDirtyActionValues_treeArtifactWithArchivedArtifact_reportsOnlyChangedKey()
throws Exception {
SpecialArtifact unchangedTreeArtifact = createTreeArtifact("dir1");
ActionExecutionValue unchangedValue =
actionValueWithTreeArtifacts(
ImmutableList.of(createTreeFileArtifactWithContent(unchangedTreeArtifact, "child")),
ImmutableList.of(createArchivedTreeArtifactWithContent(unchangedTreeArtifact)));
SpecialArtifact changedTreeArtifact = createTreeArtifact("dir2");
ArchivedTreeArtifact changedArchivedTreeArtifact =
createArchivedTreeArtifactWithContent(changedTreeArtifact, "old content");
ActionExecutionValue changedValue =
actionValueWithTreeArtifacts(
ImmutableList.of(
createTreeFileArtifactWithContent(changedTreeArtifact, "file", "content")),
ImmutableList.of(changedArchivedTreeArtifact));
writeFile(changedArchivedTreeArtifact.getPath(), "new content");
assertThat(
getDirtyActionValues(
ImmutableMap.of(
actionLookupData(0), unchangedValue, actionLookupData(1), changedValue)))
.containsExactly(actionLookupData(1));
}
private Collection<SkyKey> getDirtyActionValues(ActionExecutionValue actionExecutionValue)
throws InterruptedException {
return getDirtyActionValues(ImmutableMap.of(ACTION_LOOKUP_DATA, actionExecutionValue));
}
private Collection<SkyKey> getDirtyActionValues(ImmutableMap<SkyKey, SkyValue> valuesMap)
throws InterruptedException {
return new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
valuesMap,
batchStat.getBatchStat(fs),
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.IGNORE_ALL,
(ignored, ignored2) -> {});
}
private TreeFileArtifact createTreeFileArtifactWithContent(
SpecialArtifact treeArtifact, String parentRelativePath, String... contentLines)
throws IOException {
TreeFileArtifact artifact = TreeFileArtifact.createTreeOutput(treeArtifact, parentRelativePath);
writeFile(artifact.getPath(), contentLines);
return artifact;
}
private ArchivedTreeArtifact createArchivedTreeArtifactWithContent(
SpecialArtifact treeArtifact, String... contentLines) throws IOException {
ArchivedTreeArtifact artifact = ArchivedTreeArtifact.createForTree(treeArtifact);
writeFile(artifact.getPath(), contentLines);
return artifact;
}
private static ActionLookupData actionLookupData(int actionIndex) {
return ActionLookupData.create(ACTION_LOOKUP_KEY, actionIndex);
}
@Test
public void testEmpty() throws Exception {
FilesystemValueChecker checker =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST);
assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
}
@Test
public void testSimple() throws Exception {
FilesystemValueChecker checker =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST);
Path path = fs.getPath("/foo");
FileSystemUtils.createEmptyFile(path);
assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
SkyKey skyKey =
FileStateValue.key(
RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo")));
EvaluationResult<SkyValue> result =
evaluator.evaluate(ImmutableList.of(skyKey), EVALUATION_OPTIONS);
assertThat(result.hasError()).isFalse();
assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
FileSystemUtils.writeContentAsLatin1(path, "hello");
assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), skyKey);
// The dirty bits are not reset until the FileValues are actually revalidated.
assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), skyKey);
differencer.invalidate(ImmutableList.of(skyKey));
result = evaluator.evaluate(ImmutableList.of(skyKey), EVALUATION_OPTIONS);
assertThat(result.hasError()).isFalse();
assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
}
/**
* Tests that an already-invalidated value can still be marked changed: symlink points at sym1.
* Invalidate symlink by changing sym1 from pointing at path to point to sym2. This only dirties
* (rather than changes) symlink because sym2 still points at path, so all symlink stats remain
* the same. Then do a null build, change sym1 back to point at path, and change symlink to not be
* a symlink anymore. The fact that it is not a symlink should be detected.
*/
@Test
public void testDirtySymlink() throws Exception {
FilesystemValueChecker checker =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST);
Path path = fs.getPath("/foo");
FileSystemUtils.writeContentAsLatin1(path, "foo contents");
// We need the intermediate sym1 and sym2 so that we can dirty a child of symlink without
// actually changing the FileValue calculated for symlink (if we changed the contents of foo,
// the FileValue created for symlink would notice, since it stats foo).
Path sym1 = fs.getPath("/sym1");
Path sym2 = fs.getPath("/sym2");
Path symlink = fs.getPath("/bar");
FileSystemUtils.ensureSymbolicLink(symlink, sym1);
FileSystemUtils.ensureSymbolicLink(sym1, path);
FileSystemUtils.ensureSymbolicLink(sym2, path);
SkyKey fooKey =
FileValue.key(RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo")));
RootedPath symlinkRootedPath =
RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/bar"));
SkyKey symlinkKey = FileValue.key(symlinkRootedPath);
SkyKey symlinkFileStateKey = FileStateValue.key(symlinkRootedPath);
RootedPath sym1RootedPath =
RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/sym1"));
SkyKey sym1FileStateKey = FileStateValue.key(sym1RootedPath);
Iterable<SkyKey> allKeys = ImmutableList.of(symlinkKey, fooKey);
// First build -- prime the graph.
EvaluationResult<FileValue> result = evaluator.evaluate(allKeys, EVALUATION_OPTIONS);
assertThat(result.hasError()).isFalse();
FileValue symlinkValue = result.get(symlinkKey);
FileValue fooValue = result.get(fooKey);
assertWithMessage(symlinkValue.toString()).that(symlinkValue.isSymlink()).isTrue();
// Digest is not always available, so use size as a proxy for contents.
assertThat(symlinkValue.getSize()).isEqualTo(fooValue.getSize());
assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
// Before second build, move sym1 to point to sym2.
assertThat(sym1.delete()).isTrue();
FileSystemUtils.ensureSymbolicLink(sym1, sym2);
assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), sym1FileStateKey);
differencer.invalidate(ImmutableList.of(sym1FileStateKey));
result = evaluator.evaluate(ImmutableList.of(), EVALUATION_OPTIONS);
assertThat(result.hasError()).isFalse();
assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), sym1FileStateKey);
// Before third build, move sym1 back to original (so change pruning will prevent signaling of
// its parents, but change symlink for real.
assertThat(sym1.delete()).isTrue();
FileSystemUtils.ensureSymbolicLink(sym1, path);
assertThat(symlink.delete()).isTrue();
FileSystemUtils.writeContentAsLatin1(symlink, "new symlink contents");
assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), symlinkFileStateKey);
differencer.invalidate(ImmutableList.of(symlinkFileStateKey));
result = evaluator.evaluate(allKeys, EVALUATION_OPTIONS);
assertThat(result.hasError()).isFalse();
symlinkValue = result.get(symlinkKey);
assertWithMessage(symlinkValue.toString()).that(symlinkValue.isSymlink()).isFalse();
assertThat(result.get(fooKey)).isEqualTo(fooValue);
assertThat(symlinkValue.getSize()).isNotEqualTo(fooValue.getSize());
assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
}
@Test
public void testExplicitFiles() throws Exception {
FilesystemValueChecker checker =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST);
Path path1 = fs.getPath("/foo1");
Path path2 = fs.getPath("/foo2");
FileSystemUtils.createEmptyFile(path1);
FileSystemUtils.createEmptyFile(path2);
assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
SkyKey key1 =
FileStateValue.key(
RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo1")));
SkyKey key2 =
FileStateValue.key(
RootedPath.toRootedPath(Root.absoluteRoot(fs), PathFragment.create("/foo2")));
Iterable<SkyKey> skyKeys = ImmutableList.of(key1, key2);
EvaluationResult<SkyValue> result = evaluator.evaluate(skyKeys, EVALUATION_OPTIONS);
assertThat(result.hasError()).isFalse();
assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
// Wait for the timestamp granularity to elapse, so updating the files will observably advance
// their ctime.
TimestampGranularityUtils.waitForTimestampGranularity(
System.currentTimeMillis(), OutErr.SYSTEM_OUT_ERR);
// Update path1's contents. This will update the file's ctime with current time indicated by the
// clock.
fs.advanceClockMillis(1);
FileSystemUtils.writeContentAsLatin1(path1, "hello1");
// Update path2's mtime but not its contents. We expect that an mtime change suffices to update
// the ctime.
path2.setLastModifiedTime(42);
// Assert that both files changed. The change detection relies, among other things, on ctime
// change.
assertDiffWithNewValues(getDirtyFilesystemKeys(evaluator, checker), key1, key2);
differencer.invalidate(skyKeys);
result = evaluator.evaluate(skyKeys, EVALUATION_OPTIONS);
assertThat(result.hasError()).isFalse();
assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
}
@Test
public void testFileWithIOExceptionNotConsideredDirty() throws Exception {
Path path = fs.getPath("/testroot/foo");
path.getParentDirectory().createDirectory();
path.createSymbolicLink(PathFragment.create("bar"));
fs.readlinkThrowsIoException = true;
SkyKey fileKey =
FileStateValue.key(
RootedPath.toRootedPath(Root.fromPath(pkgRoot), PathFragment.create("foo")));
EvaluationResult<SkyValue> result =
evaluator.evaluate(ImmutableList.of(fileKey), EVALUATION_OPTIONS);
assertThat(result.hasError()).isTrue();
fs.readlinkThrowsIoException = false;
FilesystemValueChecker checker =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST);
Diff diff = getDirtyFilesystemKeys(evaluator, checker);
assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
assertThat(diff.changedKeysWithNewValues()).isEmpty();
}
@Test
public void testFilesInCycleNotConsideredDirty() throws Exception {
Path path1 = pkgRoot.getRelative("foo1");
Path path2 = pkgRoot.getRelative("foo2");
Path path3 = pkgRoot.getRelative("foo3");
FileSystemUtils.ensureSymbolicLink(path1, path2);
FileSystemUtils.ensureSymbolicLink(path2, path3);
FileSystemUtils.ensureSymbolicLink(path3, path1);
SkyKey fileKey1 = FileValue.key(RootedPath.toRootedPath(Root.fromPath(pkgRoot), path1));
EvaluationResult<SkyValue> result =
evaluator.evaluate(ImmutableList.of(fileKey1), EVALUATION_OPTIONS);
assertThat(result.hasError()).isTrue();
FilesystemValueChecker checker =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST);
Diff diff = getDirtyFilesystemKeys(evaluator, checker);
assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
assertThat(diff.changedKeysWithNewValues()).isEmpty();
}
public void checkDirtyActions(BatchStat batchStatter) throws Exception {
Artifact out1 = createDerivedArtifact("fiz");
Artifact out2 = createDerivedArtifact("pop");
FileSystemUtils.writeContentAsLatin1(out1.getPath(), "hello");
FileSystemUtils.writeContentAsLatin1(out2.getPath(), "fizzlepop");
TimestampGranularityMonitor tsgm = new TimestampGranularityMonitor(BlazeClock.instance());
SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
pretendBuildTwoArtifacts(out1, actionKey1, out2, actionKey2, batchStatter, tsgm);
// Change the file but not its size
FileSystemUtils.writeContentAsLatin1(out1.getPath(), "hallo");
checkActionDirtiedByFile(out1, actionKey1, batchStatter, tsgm);
pretendBuildTwoArtifacts(out1, actionKey1, out2, actionKey2, batchStatter, tsgm);
// Now try with a different size
FileSystemUtils.writeContentAsLatin1(out1.getPath(), "hallo2");
checkActionDirtiedByFile(out1, actionKey1, batchStatter, tsgm);
}
private void pretendBuildTwoArtifacts(
Artifact out1,
SkyKey actionKey1,
Artifact out2,
SkyKey actionKey2,
BatchStat batchStatter,
TimestampGranularityMonitor tsgm)
throws InterruptedException {
EvaluationContext evaluationContext =
EvaluationContext.newBuilder()
.setKeepGoing(false)
.setParallelism(1)
.setEventHandler(NullEventHandler.INSTANCE)
.build();
tsgm.setCommandStartTime();
differencer.inject(
ImmutableMap.of(
actionKey1,
Delta.justNew(
actionValue(
new TestAction(
Runnables.doNothing(),
NestedSetBuilder.emptySet(Order.STABLE_ORDER),
ImmutableSet.of(out1)))),
actionKey2,
Delta.justNew(
actionValue(
new TestAction(
Runnables.doNothing(),
NestedSetBuilder.emptySet(Order.STABLE_ORDER),
ImmutableSet.of(out2))))));
assertThat(evaluator.evaluate(ImmutableList.of(), evaluationContext).hasError()).isFalse();
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStatter,
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.IGNORE_ALL,
(ignored, ignored2) -> {}))
.isEmpty();
tsgm.waitForTimestampGranularity(OutErr.SYSTEM_OUT_ERR);
}
private void checkActionDirtiedByFile(
Artifact file, SkyKey actionKey, BatchStat batchStatter, TimestampGranularityMonitor tsgm)
throws InterruptedException {
assertThat(
new FilesystemValueChecker(tsgm, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStatter,
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.IGNORE_ALL,
(ignored, ignored2) -> {}))
.containsExactly(actionKey);
assertThat(
new FilesystemValueChecker(tsgm, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStatter,
ModifiedFileSet.EVERYTHING_DELETED,
RemoteArtifactChecker.IGNORE_ALL,
(ignored, ignored2) -> {}))
.containsExactly(actionKey);
assertThat(
new FilesystemValueChecker(tsgm, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStatter,
new ModifiedFileSet.Builder().modify(file.getExecPath()).build(),
RemoteArtifactChecker.IGNORE_ALL,
(ignored, ignored2) -> {}))
.containsExactly(actionKey);
assertThat(
new FilesystemValueChecker(tsgm, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStatter,
new ModifiedFileSet.Builder()
.modify(file.getExecPath().getParentDirectory())
.build(),
RemoteArtifactChecker.IGNORE_ALL,
(ignored, ignored2) -> {}))
.isEmpty();
assertThat(
new FilesystemValueChecker(tsgm, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStatter,
ModifiedFileSet.NOTHING_MODIFIED,
RemoteArtifactChecker.IGNORE_ALL,
(ignored, ignored2) -> {}))
.isEmpty();
}
enum ModifiedSetReporting {
EVERYTHING_MODIFIED {
@Override
ModifiedFileSet getModifiedFileSet(PathFragment path) {
return ModifiedFileSet.EVERYTHING_MODIFIED;
}
},
EVERYTHING_DELETED {
@Override
ModifiedFileSet getModifiedFileSet(PathFragment path) {
return ModifiedFileSet.EVERYTHING_DELETED;
}
},
SINGLE_PATH {
@Override
ModifiedFileSet getModifiedFileSet(PathFragment path) {
return ModifiedFileSet.builder().modify(path).build();
}
};
abstract ModifiedFileSet getModifiedFileSet(PathFragment path);
}
@Test
public void getDirtyActionValues_touchedTreeDirectory_returnsEmptyDiff(
@TestParameter({"", "subdir"}) String touchedTreePath,
@TestParameter ModifiedSetReporting modifiedSet)
throws Exception {
SpecialArtifact tree = createTreeArtifact("tree");
TreeFileArtifact treeFile = TreeFileArtifact.createTreeOutput(tree, "subdir/file");
FileSystemUtils.writeIsoLatin1(treeFile.getPath(), "text");
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
differencer.inject(
ImmutableMap.of(actionKey, actionValueWithTreeArtifacts(ImmutableList.of(treeFile))));
evaluate();
FileSystemUtils.touchFile(tree.getPath().getRelative(touchedTreePath));
Collection<SkyKey> dirtyActionKeys =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStat.getBatchStat(fs),
modifiedSet.getModifiedFileSet(tree.getExecPath()),
RemoteArtifactChecker.IGNORE_ALL,
mockModifiedOutputsReceiver);
assertThat(dirtyActionKeys).isEmpty();
assertThat(modifiedOutputsCaptor.getAllValues()).isEmpty();
}
@Test
public void getDirtyActionValues_deleteEmptyTreeDirectory_returnsTreeKey(
@TestParameter ModifiedSetReporting modifiedSet) throws Exception {
SpecialArtifact tree = createTreeArtifact("tree");
tree.getPath().createDirectoryAndParents();
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
differencer.inject(
ImmutableMap.of(actionKey, actionValueWithTreeArtifact(tree, TreeArtifactValue.empty())));
evaluate();
assertThat(tree.getPath().delete()).isTrue();
Collection<SkyKey> dirtyActionKeys =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStat.getBatchStat(fs),
modifiedSet.getModifiedFileSet(tree.getExecPath()),
RemoteArtifactChecker.IGNORE_ALL,
mockModifiedOutputsReceiver);
assertThat(dirtyActionKeys).containsExactly(actionKey);
assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(tree);
}
@Test
public void getDirtyActionValues_treeDirectoryReplacedWithSymlink_returnsTreeKey()
throws Exception {
SpecialArtifact tree = createTreeArtifact("tree");
tree.getPath().createDirectoryAndParents();
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
differencer.inject(
ImmutableMap.of(actionKey, actionValueWithTreeArtifact(tree, TreeArtifactValue.empty())));
evaluate();
Path dummyEmptyDir = fs.getPath("/bin").getRelative("dir");
dummyEmptyDir.createDirectoryAndParents();
assertThat(tree.getPath().delete()).isTrue();
tree.getPath().createSymbolicLink(dummyEmptyDir);
Collection<SkyKey> dirtyActionKeys =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStat.getBatchStat(fs),
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.IGNORE_ALL,
mockModifiedOutputsReceiver);
assertThat(dirtyActionKeys).containsExactly(actionKey);
assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(tree);
}
@Test
public void getDirtyActionValues_modifiedTreeFile_returnsTreeKey(
@TestParameter ModifiedSetReporting modifiedSet) throws Exception {
SpecialArtifact tree = createTreeArtifact("tree");
TreeFileArtifact treeFile = TreeFileArtifact.createTreeOutput(tree, "file");
FileSystemUtils.writeIsoLatin1(treeFile.getPath(), "text");
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
differencer.inject(
ImmutableMap.of(actionKey, actionValueWithTreeArtifacts(ImmutableList.of(treeFile))));
evaluate();
FileSystemUtils.writeIsoLatin1(treeFile.getPath(), "other text");
Collection<SkyKey> dirtyActionKeys =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStat.getBatchStat(fs),
modifiedSet.getModifiedFileSet(treeFile.getExecPath()),
RemoteArtifactChecker.IGNORE_ALL,
mockModifiedOutputsReceiver);
assertThat(dirtyActionKeys).containsExactly(actionKey);
assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(treeFile);
}
@Test
public void getDirtyActionValues_addedTreeFile_returnsTreeKey(
@TestParameter ModifiedSetReporting modifiedSet) throws Exception {
SpecialArtifact tree = createTreeArtifact("tree");
TreeFileArtifact treeFile = TreeFileArtifact.createTreeOutput(tree, "file1");
FileSystemUtils.writeIsoLatin1(treeFile.getPath());
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
differencer.inject(
ImmutableMap.of(actionKey, actionValueWithTreeArtifacts(ImmutableList.of(treeFile))));
evaluate();
TreeFileArtifact newFile = TreeFileArtifact.createTreeOutput(tree, "file2");
FileSystemUtils.writeIsoLatin1(newFile.getPath());
Collection<SkyKey> dirtyActionValues =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStat.getBatchStat(fs),
modifiedSet.getModifiedFileSet(newFile.getExecPath()),
RemoteArtifactChecker.IGNORE_ALL,
mockModifiedOutputsReceiver);
assertThat(dirtyActionValues).containsExactly(actionKey);
assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(tree);
}
@Test
public void getDirtyActionValues_addedTreeFileToEmptyTree_returnsTreeKey(
@TestParameter ModifiedSetReporting modifiedSet) throws Exception {
SpecialArtifact tree = createTreeArtifact("tree");
tree.getPath().createDirectoryAndParents();
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
differencer.inject(
ImmutableMap.of(actionKey, actionValueWithTreeArtifact(tree, TreeArtifactValue.empty())));
evaluate();
TreeFileArtifact newFile = TreeFileArtifact.createTreeOutput(tree, "file");
FileSystemUtils.writeIsoLatin1(newFile.getPath());
Collection<SkyKey> dirtyActionKeys =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStat.getBatchStat(fs),
modifiedSet.getModifiedFileSet(newFile.getExecPath()),
RemoteArtifactChecker.IGNORE_ALL,
mockModifiedOutputsReceiver);
assertThat(dirtyActionKeys).containsExactly(actionKey);
assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(tree);
}
@Test
public void getDirtyActionValues_deletedTreeFile_returnsTreeKey(
@TestParameter ModifiedSetReporting modifiedSet) throws Exception {
SpecialArtifact tree = createTreeArtifact("tree");
TreeFileArtifact treeFile = TreeFileArtifact.createTreeOutput(tree, "file");
FileSystemUtils.writeIsoLatin1(treeFile.getPath());
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
differencer.inject(
ImmutableMap.of(actionKey, actionValueWithTreeArtifacts(ImmutableList.of(treeFile))));
evaluate();
assertThat(treeFile.getPath().delete()).isTrue();
Collection<SkyKey> dirtyActionKeys =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStat.getBatchStat(fs),
modifiedSet.getModifiedFileSet(treeFile.getExecPath()),
RemoteArtifactChecker.IGNORE_ALL,
mockModifiedOutputsReceiver);
assertThat(dirtyActionKeys).containsExactly(actionKey);
assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(treeFile, tree);
}
@Test
public void getDirtyActionValues_everythingModified_returnsAllKeys() throws Exception {
SpecialArtifact tree1 = createTreeArtifact("tree1");
TreeFileArtifact tree1File = TreeFileArtifact.createTreeOutput(tree1, "file");
FileSystemUtils.writeIsoLatin1(tree1File.getPath(), "text");
SpecialArtifact tree2 = createTreeArtifact("tree2");
TreeFileArtifact tree2File = TreeFileArtifact.createTreeOutput(tree2, "file");
FileSystemUtils.writeIsoLatin1(tree2File.getPath());
SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
differencer.inject(
ImmutableMap.of(
actionKey1,
actionValueWithTreeArtifacts(ImmutableList.of(tree1File)),
actionKey2,
actionValueWithTreeArtifacts(ImmutableList.of(tree2File))));
evaluate();
FileSystemUtils.writeIsoLatin1(tree1File.getPath(), "new text");
assertThat(tree2File.getPath().delete()).isTrue();
Collection<SkyKey> dirtyActionKeys =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStat.getBatchStat(fs),
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.IGNORE_ALL,
mockModifiedOutputsReceiver);
assertThat(dirtyActionKeys).containsExactly(actionKey1, actionKey2);
assertThat(modifiedOutputsCaptor.getAllValues()).containsExactly(tree1File, tree2, tree2File);
}
@Test
public void getDirtyActionValues_changedFileNotInModifiedSet_returnsKeysFromSetOnly(
@TestParameter boolean reportFirst) throws Exception {
SpecialArtifact tree1 = createTreeArtifact("tree1");
TreeFileArtifact tree1File = TreeFileArtifact.createTreeOutput(tree1, "file");
FileSystemUtils.writeIsoLatin1(tree1File.getPath(), "text");
SpecialArtifact tree2 = createTreeArtifact("tree2");
TreeFileArtifact tree2File = TreeFileArtifact.createTreeOutput(tree2, "file");
FileSystemUtils.writeIsoLatin1(tree2File.getPath());
SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
differencer.inject(
ImmutableMap.of(
actionKey1,
actionValueWithTreeArtifacts(ImmutableList.of(tree1File)),
actionKey2,
actionValueWithTreeArtifacts(ImmutableList.of(tree2File))));
evaluate();
FileSystemUtils.writeIsoLatin1(tree1File.getPath(), "new text");
assertThat(tree2File.getPath().delete()).isTrue();
Collection<SkyKey> dirtyActionKeys =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStat.getBatchStat(fs),
ModifiedFileSet.builder()
.modify((reportFirst ? tree1File : tree2File).getExecPath())
.build(),
RemoteArtifactChecker.IGNORE_ALL,
mockModifiedOutputsReceiver);
assertThat(dirtyActionKeys).containsExactly(reportFirst ? actionKey1 : actionKey2);
assertThat(modifiedOutputsCaptor.getAllValues())
.containsExactlyElementsIn(
reportFirst ? ImmutableList.of(tree1File) : ImmutableList.of(tree2File, tree2));
}
@Test
public void getDirtyActionValues_middleFileSkippedInModifiedFileSet_returnsKeysFromSetOnly()
throws Exception {
SpecialArtifact treeA = createTreeArtifact("a_tree");
TreeFileArtifact treeAFile = TreeFileArtifact.createTreeOutput(treeA, "file");
FileSystemUtils.writeIsoLatin1(treeAFile.getPath());
SpecialArtifact treeB = createTreeArtifact("b_tree");
TreeFileArtifact treeBFile = TreeFileArtifact.createTreeOutput(treeB, "file");
FileSystemUtils.writeIsoLatin1(treeBFile.getPath());
SpecialArtifact treeC = createTreeArtifact("c_tree");
TreeFileArtifact treeCFile = TreeFileArtifact.createTreeOutput(treeC, "file");
FileSystemUtils.writeIsoLatin1(treeCFile.getPath());
SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
SkyKey actionKey3 = ActionLookupData.create(ACTION_LOOKUP_KEY, 2);
differencer.inject(
ImmutableMap.of(
actionKey1,
actionValueWithTreeArtifacts(ImmutableList.of(treeAFile)),
actionKey2,
actionValueWithTreeArtifacts(ImmutableList.of(treeBFile)),
actionKey3,
actionValueWithTreeArtifacts(ImmutableList.of(treeCFile))));
evaluate();
assertThat(treeAFile.getPath().delete()).isTrue();
assertThat(treeBFile.getPath().delete()).isTrue();
assertThat(treeCFile.getPath().delete()).isTrue();
Collection<SkyKey> dirtyActionKeys =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStat.getBatchStat(fs),
ModifiedFileSet.builder()
.modify(treeAFile.getExecPath())
.modify(treeCFile.getExecPath())
.build(),
RemoteArtifactChecker.IGNORE_ALL,
mockModifiedOutputsReceiver);
assertThat(dirtyActionKeys).containsExactly(actionKey1, actionKey3);
assertThat(modifiedOutputsCaptor.getAllValues())
.containsExactly(treeAFile, treeA, treeCFile, treeC);
}
@Test
public void getDirtyActionValues_nothingModified_returnsEmptyDiff() throws Exception {
SpecialArtifact tree = createTreeArtifact("tree");
TreeFileArtifact treeFile = TreeFileArtifact.createTreeOutput(tree, "file");
FileSystemUtils.writeIsoLatin1(treeFile.getPath());
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
differencer.inject(
ImmutableMap.of(actionKey, actionValueWithTreeArtifacts(ImmutableList.of(treeFile))));
evaluate();
assertThat(treeFile.getPath().delete()).isTrue();
Collection<SkyKey> dirtyActionKeys =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
batchStat.getBatchStat(fs),
ModifiedFileSet.NOTHING_MODIFIED,
RemoteArtifactChecker.IGNORE_ALL,
mockModifiedOutputsReceiver);
assertThat(dirtyActionKeys).isEmpty();
assertThat(modifiedOutputsCaptor.getAllValues()).isEmpty();
}
private void evaluate() throws InterruptedException {
EvaluationContext evaluationContext =
EvaluationContext.newBuilder()
.setKeepGoing(false)
.setParallelism(1)
.setEventHandler(NullEventHandler.INSTANCE)
.build();
assertThat(evaluator.evaluate(ImmutableList.of(), evaluationContext).hasError()).isFalse();
}
private Artifact createDerivedArtifact(String relPath) throws IOException {
String outSegment = "bin";
Path outputPath = fs.getPath("/" + outSegment);
outputPath.createDirectory();
return ActionsTestUtil.createArtifact(
ArtifactRoot.asDerivedRoot(fs.getPath("/"), RootType.Output, outSegment),
outputPath.getRelative(relPath));
}
@Test
// TODO(b/154337187): Remove the following annotation to re-enable once this test is de-flaked.
@Ignore("b/154337187")
public void testDirtyActions() throws Exception {
checkDirtyActions(null);
}
@Test
// TODO(b/154337187): Remove the following annotation to re-enable once this test is de-flaked.
@Ignore("b/154337187")
public void testDirtyActionsBatchStat() throws Exception {
checkDirtyActions(
new BatchStat() {
@Override
public List<FileStatusWithDigest> batchStat(
boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
throws IOException {
List<FileStatusWithDigest> stats = new ArrayList<>();
for (PathFragment pathFrag : paths) {
stats.add(
FileStatusWithDigestAdapter.maybeAdapt(
fs.getPath("/").getRelative(pathFrag).statIfFound(Symlinks.NOFOLLOW)));
}
return stats;
}
});
}
@Test
// TODO(b/154337187): Remove the following annotation to re-enable once this test is de-flaked.
@Ignore("b/154337187")
public void testDirtyActionsBatchStatWithDigest() throws Exception {
checkDirtyActions(
new BatchStat() {
@Override
public List<FileStatusWithDigest> batchStat(
boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
throws IOException {
List<FileStatusWithDigest> stats = new ArrayList<>();
for (PathFragment pathFrag : paths) {
final Path path = fs.getPath("/").getRelative(pathFrag);
stats.add(statWithDigest(path, path.statIfFound(Symlinks.NOFOLLOW)));
}
return stats;
}
});
}
@Test
// TODO(b/154337187): Remove the following annotation to re-enable once this test is de-flaked.
@Ignore("b/154337187")
public void testDirtyActionsBatchStatFallback() throws Exception {
checkDirtyActions(
new BatchStat() {
@Override
public List<FileStatusWithDigest> batchStat(
boolean useDigest, boolean includeLinks, Iterable<PathFragment> paths)
throws IOException {
throw new IOException("try again");
}
});
}
// TODO(bazel-team): Add some tests for FileSystemValueChecker#changedKeys*() methods.
// Presently these appear to be untested.
private static ActionExecutionValue actionValue(Action action) {
Map<Artifact, FileArtifactValue> artifactData = new HashMap<>();
for (Artifact output : action.getOutputs()) {
try {
Path path = output.getPath();
FileArtifactValue noDigest =
ActionMetadataHandler.fileArtifactValueFromArtifact(
output,
FileStatusWithDigestAdapter.maybeAdapt(path.statIfFound(Symlinks.NOFOLLOW)),
SyscallCache.NO_CACHE,
null);
FileArtifactValue withDigest =
FileArtifactValue.createFromInjectedDigest(noDigest, path.getDigest());
artifactData.put(output, withDigest);
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
return ActionsTestUtil.createActionExecutionValue(ImmutableMap.copyOf(artifactData));
}
private static Delta actionValueWithTreeArtifact(SpecialArtifact output, TreeArtifactValue tree) {
return Delta.justNew(
ActionsTestUtil.createActionExecutionValue(
/* artifactData= */ ImmutableMap.of(), ImmutableMap.of(output, tree)));
}
private static Delta actionValueWithRemoteArtifact(
Artifact output, RemoteFileArtifactValue value) {
return Delta.justNew(
ActionsTestUtil.createActionExecutionValue(ImmutableMap.of(output, value)));
}
private RemoteFileArtifactValue createRemoteFileArtifactValue(String contents) {
return createRemoteFileArtifactValue(contents, /* expireAtEpochMilli= */ -1);
}
private RemoteFileArtifactValue createRemoteFileArtifactValue(
String contents, long expireAtEpochMilli) {
byte[] data = contents.getBytes();
DigestHashFunction hashFn = fs.getDigestFunction();
HashCode hash = hashFn.getHashFunction().hashBytes(data);
return RemoteFileArtifactValue.create(hash.asBytes(), data.length, -1, expireAtEpochMilli);
}
@Test
public void testRemoteAndLocalArtifacts() throws Exception {
// Test that injected remote artifacts are trusted by the FileSystemValueChecker
// if it is configured to trust remote artifacts, and that local files always take precedence
// over remote files.
SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
Artifact out1 = createDerivedArtifact("foo");
Artifact out2 = createDerivedArtifact("bar");
Map<SkyKey, Delta> metadataToInject = new HashMap<>();
metadataToInject.put(
actionKey1,
actionValueWithRemoteArtifact(out1, createRemoteFileArtifactValue("foo-content")));
metadataToInject.put(
actionKey2,
actionValueWithRemoteArtifact(out2, createRemoteFileArtifactValue("bar-content")));
differencer.inject(metadataToInject);
EvaluationContext evaluationContext =
EvaluationContext.newBuilder()
.setKeepGoing(false)
.setParallelism(1)
.setEventHandler(NullEventHandler.INSTANCE)
.build();
assertThat(
evaluator
.evaluate(ImmutableList.of(actionKey1, actionKey2), evaluationContext)
.hasError())
.isFalse();
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
/* batchStatter= */ null,
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.TRUST_ALL,
(ignored, ignored2) -> {}))
.isEmpty();
// Create the "out1" artifact on the filesystem and test that it invalidates the generating
// action's SkyKey.
FileSystemUtils.writeContentAsLatin1(out1.getPath(), "new-foo-content");
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
/* batchStatter= */ null,
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.TRUST_ALL,
(ignored, ignored2) -> {}))
.containsExactly(actionKey1);
}
@Test
public void testRemoteAndLocalArtifacts_identicalContent() throws Exception {
// Test that even if injected remote artifacts and local files are identical, the generating
// actions are marked as dirty.
SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
Artifact out1 = createDerivedArtifact("foo");
Artifact out2 = createDerivedArtifact("bar");
Map<SkyKey, Delta> metadataToInject = new HashMap<>();
metadataToInject.put(
actionKey1,
actionValueWithRemoteArtifact(out1, createRemoteFileArtifactValue("foo-content")));
metadataToInject.put(
actionKey2,
actionValueWithRemoteArtifact(out2, createRemoteFileArtifactValue("bar-content")));
differencer.inject(metadataToInject);
EvaluationContext evaluationContext =
EvaluationContext.newBuilder()
.setKeepGoing(false)
.setParallelism(1)
.setEventHandler(NullEventHandler.INSTANCE)
.build();
assertThat(
evaluator
.evaluate(ImmutableList.of(actionKey1, actionKey2), evaluationContext)
.hasError())
.isFalse();
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
/* batchStatter= */ null,
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.TRUST_ALL,
(ignored, ignored2) -> {}))
.isEmpty();
// Create identical "out1" artifact on the filesystem and test that it invalidates the
// generating action's SkyKey.
FileSystemUtils.writeContentAsLatin1(out1.getPath(), "foo-content");
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
/* batchStatter= */ null,
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.TRUST_ALL,
(ignored, ignored2) -> {}))
.containsExactly(actionKey1);
}
@Test
public void testRemoteArtifactsExpired() throws Exception {
// Test that if injected remote artifacts expired, they are considered as dirty.
SkyKey actionKey1 = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
SkyKey actionKey2 = ActionLookupData.create(ACTION_LOOKUP_KEY, 1);
Artifact out1 = createDerivedArtifact("foo");
Artifact out2 = createDerivedArtifact("bar");
Map<SkyKey, Delta> metadataToInject = new HashMap<>();
metadataToInject.put(
actionKey1,
actionValueWithRemoteArtifact(out1, createRemoteFileArtifactValue("foo-content")));
metadataToInject.put(
actionKey2,
actionValueWithRemoteArtifact(
out2, createRemoteFileArtifactValue("bar-content", /* expireAtEpochMilli= */ 0)));
differencer.inject(metadataToInject);
EvaluationContext evaluationContext =
EvaluationContext.newBuilder()
.setKeepGoing(false)
.setParallelism(1)
.setEventHandler(NullEventHandler.INSTANCE)
.build();
assertThat(
evaluator
.evaluate(ImmutableList.of(actionKey1, actionKey2), evaluationContext)
.hasError())
.isFalse();
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
/* batchStatter= */ null,
ModifiedFileSet.EVERYTHING_MODIFIED,
CHECK_TTL,
(ignored, ignored2) -> {}))
.containsExactly(actionKey2);
}
@Test
public void testRemoteAndLocalTreeArtifacts() throws Exception {
// Test that injected remote tree artifacts are trusted by the FileSystemValueChecker
// and that local files always takes preference over remote files.
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
SpecialArtifact treeArtifact = createTreeArtifact("dir");
treeArtifact.getPath().createDirectoryAndParents();
TreeArtifactValue tree =
TreeArtifactValue.newBuilder(treeArtifact)
.putChild(
TreeFileArtifact.createTreeOutput(treeArtifact, "foo"),
createRemoteFileArtifactValue("foo-content"))
.putChild(
TreeFileArtifact.createTreeOutput(treeArtifact, "bar"),
createRemoteFileArtifactValue("bar-content"))
.build();
differencer.inject(ImmutableMap.of(actionKey, actionValueWithTreeArtifact(treeArtifact, tree)));
EvaluationContext evaluationContext =
EvaluationContext.newBuilder()
.setKeepGoing(false)
.setParallelism(1)
.setEventHandler(NullEventHandler.INSTANCE)
.build();
assertThat(evaluator.evaluate(ImmutableList.of(actionKey), evaluationContext).hasError())
.isFalse();
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
/* batchStatter= */ null,
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.TRUST_ALL,
(ignored, ignored2) -> {}))
.isEmpty();
// Create dir/foo on the local disk and test that it invalidates the associated sky key.
TreeFileArtifact fooArtifact = TreeFileArtifact.createTreeOutput(treeArtifact, "foo");
FileSystemUtils.writeContentAsLatin1(fooArtifact.getPath(), "new-foo-content");
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
/* batchStatter= */ null,
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.IGNORE_ALL,
(ignored, ignored2) -> {}))
.containsExactly(actionKey);
}
@Test
public void testRemoteAndLocalTreeArtifacts_identicalContent() throws Exception {
// Test that even if injected remote tree artifacts and local files are identical, the
// generating actions are marked as dirty.
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
SpecialArtifact treeArtifact = createTreeArtifact("dir");
treeArtifact.getPath().createDirectoryAndParents();
TreeArtifactValue tree =
TreeArtifactValue.newBuilder(treeArtifact)
.putChild(
TreeFileArtifact.createTreeOutput(treeArtifact, "foo"),
createRemoteFileArtifactValue("foo-content"))
.putChild(
TreeFileArtifact.createTreeOutput(treeArtifact, "bar"),
createRemoteFileArtifactValue("bar-content"))
.build();
differencer.inject(ImmutableMap.of(actionKey, actionValueWithTreeArtifact(treeArtifact, tree)));
EvaluationContext evaluationContext =
EvaluationContext.newBuilder()
.setKeepGoing(false)
.setParallelism(1)
.setEventHandler(NullEventHandler.INSTANCE)
.build();
assertThat(evaluator.evaluate(ImmutableList.of(actionKey), evaluationContext).hasError())
.isFalse();
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
/* batchStatter= */ null,
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.TRUST_ALL,
(ignored, ignored2) -> {}))
.isEmpty();
// Create identical dir/foo on the local disk and test that it invalidates the associated sky
// key.
TreeFileArtifact fooArtifact = TreeFileArtifact.createTreeOutput(treeArtifact, "foo");
FileSystemUtils.writeContentAsLatin1(fooArtifact.getPath(), "foo-content");
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
/* batchStatter= */ null,
ModifiedFileSet.EVERYTHING_MODIFIED,
RemoteArtifactChecker.TRUST_ALL,
(ignored, ignored2) -> {}))
.containsExactly(actionKey);
}
@Test
public void testRemoteTreeArtifactsExpired() throws Exception {
// Test that if injected remote tree artifacts are expired, they are considered as dirty.
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
SpecialArtifact treeArtifact = createTreeArtifact("dir");
treeArtifact.getPath().createDirectoryAndParents();
TreeArtifactValue tree =
TreeArtifactValue.newBuilder(treeArtifact)
.putChild(
TreeFileArtifact.createTreeOutput(treeArtifact, "foo"),
createRemoteFileArtifactValue("foo-content"))
.putChild(
TreeFileArtifact.createTreeOutput(treeArtifact, "bar"),
createRemoteFileArtifactValue("bar-content", /* expireAtEpochMilli= */ 0))
.build();
differencer.inject(ImmutableMap.of(actionKey, actionValueWithTreeArtifact(treeArtifact, tree)));
EvaluationContext evaluationContext =
EvaluationContext.newBuilder()
.setKeepGoing(false)
.setParallelism(1)
.setEventHandler(NullEventHandler.INSTANCE)
.build();
assertThat(evaluator.evaluate(ImmutableList.of(actionKey), evaluationContext).hasError())
.isFalse();
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
/* batchStatter= */ null,
ModifiedFileSet.EVERYTHING_MODIFIED,
CHECK_TTL,
(ignored, ignored2) -> {}))
.containsExactly(actionKey);
}
@Test
public void testRemoteTreeArtifacts_archivedRepresentationExpired() throws Exception {
// Test that if archived representation of injected remote tree artifacts are expired, they are
// considered as dirty.
SkyKey actionKey = ActionLookupData.create(ACTION_LOOKUP_KEY, 0);
SpecialArtifact treeArtifact = createTreeArtifact("dir");
treeArtifact.getPath().createDirectoryAndParents();
TreeArtifactValue tree =
TreeArtifactValue.newBuilder(treeArtifact)
.putChild(
TreeFileArtifact.createTreeOutput(treeArtifact, "foo"),
createRemoteFileArtifactValue("foo-content"))
.putChild(
TreeFileArtifact.createTreeOutput(treeArtifact, "bar"),
createRemoteFileArtifactValue("bar-content"))
.setArchivedRepresentation(
createArchivedTreeArtifactWithContent(treeArtifact),
createRemoteFileArtifactValue("archived", /* expireAtEpochMilli= */ 0))
.build();
differencer.inject(ImmutableMap.of(actionKey, actionValueWithTreeArtifact(treeArtifact, tree)));
EvaluationContext evaluationContext =
EvaluationContext.newBuilder()
.setKeepGoing(false)
.setParallelism(1)
.setEventHandler(NullEventHandler.INSTANCE)
.build();
assertThat(evaluator.evaluate(ImmutableList.of(actionKey), evaluationContext).hasError())
.isFalse();
assertThat(
new FilesystemValueChecker(
/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST)
.getDirtyActionValues(
evaluator.getValues(),
/* batchStatter= */ null,
ModifiedFileSet.EVERYTHING_MODIFIED,
CHECK_TTL,
(ignored, ignored2) -> {}))
.containsExactly(actionKey);
}
@Test
public void testPropagatesRuntimeExceptions() throws Exception {
Collection<SkyKey> values =
ImmutableList.of(
FileValue.key(
RootedPath.toRootedPath(Root.fromPath(pkgRoot), PathFragment.create("foo"))));
evaluator.evaluate(values, EVALUATION_OPTIONS);
AtomicReference<Throwable> uncaughtRef = new AtomicReference<>();
CountDownLatch throwableCaught = new CountDownLatch(1);
Thread.UncaughtExceptionHandler uncaughtExceptionHandler =
(t, e) -> {
uncaughtRef.compareAndSet(null, e);
throwableCaught.countDown();
};
Thread.setDefaultUncaughtExceptionHandler(uncaughtExceptionHandler);
FilesystemValueChecker checker =
new FilesystemValueChecker(/* tsgm= */ null, SyscallCache.NO_CACHE, FSVC_THREADS_FOR_TEST);
assertEmptyDiff(getDirtyFilesystemKeys(evaluator, checker));
fs.statThrowsRuntimeException = true;
getDirtyFilesystemKeys(evaluator, checker);
// Wait for exception handler to trigger (FVC doesn't clean up crashing threads on its own).
assertThat(throwableCaught.await(TestUtils.WAIT_TIMEOUT_SECONDS, SECONDS)).isTrue();
Throwable thrown = uncaughtRef.get();
assertThat(thrown).isNotNull();
assertThat(thrown).hasMessageThat().isEqualTo("bork");
assertThat(thrown).isInstanceOf(RuntimeException.class);
}
private static void assertEmptyDiff(Diff diff) {
assertDiffWithNewValues(diff);
}
private static void assertDiffWithNewValues(Diff diff, SkyKey... keysWithNewValues) {
assertThat(diff.changedKeysWithoutNewValues()).isEmpty();
assertThat(diff.changedKeysWithNewValues().keySet())
.containsExactlyElementsIn(Arrays.asList(keysWithNewValues));
}
private static FileStatusWithDigest statWithDigest(final Path path, final FileStatus stat) {
return new FileStatusWithDigest() {
@Nullable
@Override
public byte[] getDigest() throws IOException {
return path.getDigest();
}
@Override
public boolean isFile() {
return stat.isFile();
}
@Override
public boolean isSpecialFile() {
return stat.isSpecialFile();
}
@Override
public boolean isDirectory() {
return stat.isDirectory();
}
@Override
public boolean isSymbolicLink() {
return stat.isSymbolicLink();
}
@Override
public long getSize() throws IOException {
return stat.getSize();
}
@Override
public long getLastModifiedTime() throws IOException {
return stat.getLastModifiedTime();
}
@Override
public long getLastChangeTime() throws IOException {
return stat.getLastChangeTime();
}
@Override
public long getNodeId() throws IOException {
return stat.getNodeId();
}
};
}
private static Diff getDirtyFilesystemKeys(
MemoizingEvaluator evaluator, FilesystemValueChecker checker) throws InterruptedException {
return checker.getDirtyKeys(
evaluator.getValues(), DirtinessCheckerUtils.createBasicFilesystemDirtinessChecker());
}
}