Add artifact digest and length to BEP. Various BEP consumers would like to know whether a given artifact has changed from one invocation to the next. Since we produce BEP after an action has completed, we know the digest of all inputs/outputs and can report them with the file URI. RELNOTES: The Build Event Protocol now contains file digests and sizes along with the file name and URI. PiperOrigin-RevId: 409181875
diff --git a/src/main/java/com/google/devtools/build/lib/actions/CompletionContext.java b/src/main/java/com/google/devtools/build/lib/actions/CompletionContext.java index 9adb7c1..12df48a 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/CompletionContext.java +++ b/src/main/java/com/google/devtools/build/lib/actions/CompletionContext.java
@@ -25,6 +25,7 @@ import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import java.util.Map; +import javax.annotation.Nullable; /** * Container for the data one needs to resolve aggregate artifacts from events signaling the @@ -104,16 +105,13 @@ return pathResolver; } + @Nullable + public FileArtifactValue getFileArtifactValue(Artifact artifact) { + return importantInputMap.getMetadata(artifact); + } + /** Returns true if the given artifact is guaranteed to be a file (and not a directory). */ - public boolean isGuaranteedToBeOutputFile(Artifact artifact) { - FileArtifactValue metadata = importantInputMap.getMetadata(artifact); - // If we have no metadata for an output file that will be reported in BEP, return that the - // output is not guaranteed to be a file. (We expect this to happen for baseline_coverage.dat - // files when coverage is enabled.) - if (metadata == null) { - return false; - } - FileStateType type = metadata.getType(); + public static boolean isGuaranteedToBeOutputFile(FileStateType type) { return type == FileStateType.REGULAR_FILE || type == FileStateType.SPECIAL_FILE || type == FileStateType.NONEXISTENT;
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BUILD b/src/main/java/com/google/devtools/build/lib/analysis/BUILD index 7eef655..b4084ca 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BUILD +++ b/src/main/java/com/google/devtools/build/lib/analysis/BUILD
@@ -374,6 +374,7 @@ "//src/main/java/com/google/devtools/build/lib/actions:artifacts", "//src/main/java/com/google/devtools/build/lib/actions:commandline_item", "//src/main/java/com/google/devtools/build/lib/actions:execution_requirements", + "//src/main/java/com/google/devtools/build/lib/actions:file_metadata", "//src/main/java/com/google/devtools/build/lib/actions:fileset_output_symlink", "//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity", "//src/main/java/com/google/devtools/build/lib/actions:package_roots",
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java index 9c54ee9..8fc1c54 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java
@@ -23,10 +23,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.io.BaseEncoding; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.CompletionContext; import com.google.devtools.build.lib.actions.CompletionContext.ArtifactReceiver; import com.google.devtools.build.lib.actions.EventReportingArtifacts; +import com.google.devtools.build.lib.actions.FileArtifactValue; import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper.ArtifactsInOutputGroup; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; @@ -110,6 +112,8 @@ } } + private static final BaseEncoding LOWERCASE_HEX_ENCODING = BaseEncoding.base16().lowerCase(); + private final Label label; private final ConfiguredTargetKey configuredTargetKey; private final NestedSet<Cause> rootCauses; @@ -337,7 +341,8 @@ String uri = converters.pathConverter().apply(completionContext.pathResolver().toPath(artifact)); if (uri != null) { - builder.addImportantOutput(newFileFromArtifact(name, artifact).setUri(uri).build()); + builder.addImportantOutput( + newFileFromArtifact(name, artifact, completionContext).setUri(uri).build()); } } @@ -354,12 +359,12 @@ } public static BuildEventStreamProtos.File.Builder newFileFromArtifact( - String name, Artifact artifact) { - return newFileFromArtifact(name, artifact, PathFragment.EMPTY_FRAGMENT); + String name, Artifact artifact, CompletionContext completionContext) { + return newFileFromArtifact(name, artifact, PathFragment.EMPTY_FRAGMENT, completionContext); } public static BuildEventStreamProtos.File.Builder newFileFromArtifact( - String name, Artifact artifact, PathFragment relPath) { + String name, Artifact artifact, PathFragment relPath, CompletionContext completionContext) { if (name == null) { name = artifact.getRootRelativePath().getRelative(relPath).getPathString(); if (OS.getCurrent() != OS.WINDOWS) { @@ -373,13 +378,24 @@ name = new String(name.getBytes(ISO_8859_1), UTF_8); } } - return File.newBuilder() - .setName(name) - .addAllPathPrefix(artifact.getRoot().getExecPath().segments()); + File.Builder file = + File.newBuilder() + .setName(name) + .addAllPathPrefix(artifact.getRoot().getExecPath().segments()); + FileArtifactValue fileArtifactValue = completionContext.getFileArtifactValue(artifact); + if (fileArtifactValue != null && fileArtifactValue.getType().exists()) { + byte[] digest = fileArtifactValue.getDigest(); + if (digest != null) { + file.setDigest(LOWERCASE_HEX_ENCODING.encode(digest)); + } + file.setLength(fileArtifactValue.getSize()); + } + return file; } - public static BuildEventStreamProtos.File.Builder newFileFromArtifact(Artifact artifact) { - return newFileFromArtifact(/* name= */ null, artifact); + public static BuildEventStreamProtos.File.Builder newFileFromArtifact( + Artifact artifact, CompletionContext completionContext) { + return newFileFromArtifact(/* name= */ null, artifact, completionContext); } @Override @@ -444,7 +460,7 @@ Iterable<Artifact> filteredImportantArtifacts = getLegacyFilteredImportantArtifacts(); for (Artifact artifact : filteredImportantArtifacts) { if (artifact.isDirectory()) { - builder.addDirectoryOutput(newFileFromArtifact(artifact).build()); + builder.addDirectoryOutput(newFileFromArtifact(artifact, completionContext).build()); } } // TODO(aehlig): remove direct reporting of artifacts as soon as clients no longer need it.
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto index e9e0058..6da5620 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto
@@ -479,6 +479,13 @@ // The contents of the file, if they are guaranteed to be short. bytes contents = 3; } + + // Digest of the file, using the build tool's configured digest algorithm, + // hex-encoded. + string digest = 5; + + // Length of the file in bytes. + int64 length = 6; } // Payload of a message to describe a set of files, usually build artifacts, to
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/NamedArtifactGroup.java b/src/main/java/com/google/devtools/build/lib/runtime/NamedArtifactGroup.java index 75d56fa..6c0439e 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/NamedArtifactGroup.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/NamedArtifactGroup.java
@@ -14,6 +14,7 @@ package com.google.devtools.build.lib.runtime; +import static com.google.devtools.build.lib.actions.CompletionContext.isGuaranteedToBeOutputFile; import static com.google.devtools.build.lib.analysis.TargetCompleteEvent.newFileFromArtifact; import com.google.common.collect.ImmutableList; @@ -21,6 +22,7 @@ import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.CompletionContext; import com.google.devtools.build.lib.actions.CompletionContext.ArtifactReceiver; +import com.google.devtools.build.lib.actions.FileArtifactValue; import com.google.devtools.build.lib.buildeventstream.ArtifactGroupNamer; import com.google.devtools.build.lib.buildeventstream.BuildEvent; import com.google.devtools.build.lib.buildeventstream.BuildEvent.LocalFile.LocalFileType; @@ -74,8 +76,10 @@ for (Object elem : set.getLeaves()) { ExpandedArtifact expandedArtifact = (ExpandedArtifact) elem; if (expandedArtifact.relPath == null) { + FileArtifactValue fileMetadata = + completionContext.getFileArtifactValue(expandedArtifact.artifact); LocalFileType outputType = - completionContext.isGuaranteedToBeOutputFile(expandedArtifact.artifact) + fileMetadata != null && isGuaranteedToBeOutputFile(fileMetadata.getType()) ? LocalFileType.OUTPUT_FILE : LocalFileType.OUTPUT; artifacts.add( @@ -104,7 +108,8 @@ String uri = pathConverter.apply(completionContext.pathResolver().toPath(expandedArtifact.artifact)); if (uri != null) { - builder.addFiles(newFileFromArtifact(expandedArtifact.artifact).setUri(uri)); + builder.addFiles( + newFileFromArtifact(expandedArtifact.artifact, completionContext).setUri(uri)); } } else { String uri = @@ -112,9 +117,9 @@ completionContext.pathResolver().convertPath(expandedArtifact.target)); if (uri != null) { builder.addFiles( - newFileFromArtifact(null, expandedArtifact.artifact, expandedArtifact.relPath) - .setUri(uri) - .build()); + newFileFromArtifact( + null, expandedArtifact.artifact, expandedArtifact.relPath, completionContext) + .setUri(uri)); } } }
diff --git a/src/test/java/com/google/devtools/build/lib/analysis/TargetCompleteEventTest.java b/src/test/java/com/google/devtools/build/lib/analysis/TargetCompleteEventTest.java index 4c8fd7f..653085a 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/TargetCompleteEventTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/TargetCompleteEventTest.java
@@ -15,13 +15,13 @@ package com.google.devtools.build.lib.analysis; import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.actions.CompletionContext.FAILED_COMPLETION_CTX; import static com.google.devtools.build.lib.analysis.TargetCompleteEvent.newFileFromArtifact; import static java.nio.charset.StandardCharsets.ISO_8859_1; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.Iterables; import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.CompletionContext; import com.google.devtools.build.lib.actions.EventReportingArtifacts.ReportedArtifacts; import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper.ArtifactsToBuild; import com.google.devtools.build.lib.analysis.util.AnalysisTestCase; @@ -65,7 +65,7 @@ TargetCompleteEvent event = TargetCompleteEvent.successfulBuild( ctAndData, - CompletionContext.FAILED_COMPLETION_CTX, + FAILED_COMPLETION_CTX, artifactsToBuild.getAllArtifactsByOutputGroup(), /*announceTargetSummary=*/ false); @@ -73,7 +73,9 @@ ReportedArtifacts reportedArtifacts = event.reportedArtifacts(); for (NestedSet<Artifact> artifactSet : reportedArtifacts.artifacts) { for (Artifact a : artifactSet.toListInterruptibly()) { - fileProtos.add(newFileFromArtifact(null, a, PathFragment.EMPTY_FRAGMENT).build()); + fileProtos.add( + newFileFromArtifact(null, a, PathFragment.EMPTY_FRAGMENT, FAILED_COMPLETION_CTX) + .build()); } } // Bytes are the same but the encoding is actually UTF-8 as required of a protobuf string.
diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/BUILD b/src/test/java/com/google/devtools/build/lib/buildtool/BUILD index c6f3026..b06c6b8 100644 --- a/src/test/java/com/google/devtools/build/lib/buildtool/BUILD +++ b/src/test/java/com/google/devtools/build/lib/buildtool/BUILD
@@ -551,21 +551,28 @@ ) java_test( - name = "TargetCompleteEventKeepsMinimalMetadataTest", - srcs = ["TargetCompleteEventKeepsMinimalMetadataTest.java"], + name = "TargetCompleteEventTest", + srcs = ["TargetCompleteEventTest.java"], tags = [ "no_windows", ], deps = [ "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib/actions", "//src/main/java/com/google/devtools/build/lib/actions:artifacts", + "//src/main/java/com/google/devtools/build/lib/actions:file_metadata", "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster", "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", + "//src/main/java/com/google/devtools/build/lib/buildeventservice", + "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/collect/nestedset", + "//src/main/java/com/google/devtools/build/lib/vfs", "//src/test/java/com/google/devtools/build/lib/analysis/util", "//src/test/java/com/google/devtools/build/lib/buildtool/util", + "//src/test/java/com/google/devtools/build/lib/packages:testutil", "//third_party:guava", + "//third_party:jsr305", "//third_party:junit4", "//third_party:truth", ],
diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/TargetCompleteEventKeepsMinimalMetadataTest.java b/src/test/java/com/google/devtools/build/lib/buildtool/TargetCompleteEventKeepsMinimalMetadataTest.java deleted file mode 100644 index d26ccb8..0000000 --- a/src/test/java/com/google/devtools/build/lib/buildtool/TargetCompleteEventKeepsMinimalMetadataTest.java +++ /dev/null
@@ -1,138 +0,0 @@ -// Copyright 2021 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.buildtool; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.common.collect.Iterables; -import com.google.common.eventbus.Subscribe; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.analysis.ConfiguredTarget; -import com.google.devtools.build.lib.analysis.OutputGroupInfo; -import com.google.devtools.build.lib.analysis.TargetCompleteEvent; -import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; -import com.google.devtools.build.lib.analysis.util.AnalysisMock; -import com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase; -import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.collect.nestedset.NestedSet; -import java.util.Collection; -import java.util.concurrent.atomic.AtomicReference; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** - * Validates that TargetCompleteEvents do not keep a map of action output metadata for the - * _validation output group, which can be quite large. - */ -@RunWith(JUnit4.class) -public class TargetCompleteEventKeepsMinimalMetadataTest extends BuildIntegrationTestCase { - - @Before - public void stageEmbeddedTools() throws Exception { - AnalysisMock.get().setupMockToolsRepository(mockToolsConfig); - } - - @Test - public void artifactsNotRetained() throws Exception { - write( - "validation_actions/defs.bzl", - "def _rule_with_implicit_outs_and_validation_impl(ctx):", - "", - " ctx.actions.write(ctx.outputs.main, \"main output\\n\")", - "", - " ctx.actions.write(ctx.outputs.implicit, \"implicit output\\n\")", - "", - " validation_output = ctx.actions.declare_file(ctx.attr.name + \".validation\")", - " # The actual tool will be created in individual tests, depending on whether", - " # validation should pass or fail.", - " ctx.actions.run(", - " outputs = [validation_output],", - " executable = ctx.executable._validation_tool,", - " arguments = [validation_output.path])", - "", - " return [", - " DefaultInfo(files = depset([ctx.outputs.main])),", - " OutputGroupInfo(_validation = depset([validation_output])),", - " ]", - "", - "", - "rule_with_implicit_outs_and_validation = rule(", - " implementation = _rule_with_implicit_outs_and_validation_impl,", - " outputs = {", - " \"main\": \"%{name}.main\",", - " \"implicit\": \"%{name}.implicit\",", - " },", - " attrs = {", - " \"_validation_tool\": attr.label(", - " allow_single_file = True,", - " default = Label(\"//validation_actions:validation_tool\"),", - " executable = True,", - " cfg = \"host\"),", - " }", - ")"); - write("validation_actions/validation_tool", "#!/bin/bash", "echo \"validation output\" > $1") - .setExecutable(true); - write( - "validation_actions/BUILD", - "load(", - " \":defs.bzl\",", - " \"rule_with_implicit_outs_and_validation\")", - "", - "rule_with_implicit_outs_and_validation(name = \"foo0\")"); - - AtomicReference<TargetCompleteEvent> targetCompleteEventRef = new AtomicReference<>(); - runtimeWrapper.registerSubscriber( - new Object() { - @SuppressWarnings("unused") - @Subscribe - public void accept(TargetCompleteEvent event) { - targetCompleteEventRef.set(event); - } - }); - - addOptions("--experimental_run_validations"); - BuildResult buildResult = buildTarget("//validation_actions:foo0"); - - Collection<ConfiguredTarget> successfulTargets = buildResult.getSuccessfulTargets(); - ConfiguredTarget fooTarget = Iterables.getOnlyElement(successfulTargets); - - // Check that the primary output, :foo0.main, has its metadata retained and the - // CompletionContext can confirm it is an output file. - Artifact main = - ((RuleConfiguredTarget) fooTarget) - .getArtifactByOutputLabel( - Label.parseAbsoluteUnchecked("//validation_actions:foo0.main")); - assertThat(targetCompleteEventRef.get().getCompletionContext().isGuaranteedToBeOutputFile(main)) - .isTrue(); - - // Check that the validation output, :foo0.validation, does not have its metadata retained and - // the CompletionContext cannot confirm it is an output file (even though it is). - OutputGroupInfo outputGroups = fooTarget.get(OutputGroupInfo.STARLARK_CONSTRUCTOR); - NestedSet<Artifact> validationArtifacts = - outputGroups.getOutputGroup(OutputGroupInfo.VALIDATION); - assertThat(validationArtifacts.isEmpty()).isFalse(); - - Artifact validationArtifact = Iterables.getOnlyElement(validationArtifacts.toList()); - - assertThat(targetCompleteEventRef.get()).isNotNull(); - assertThat( - targetCompleteEventRef - .get() - .getCompletionContext() - .isGuaranteedToBeOutputFile(validationArtifact)) - .isFalse(); - } -}
diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/TargetCompleteEventTest.java b/src/test/java/com/google/devtools/build/lib/buildtool/TargetCompleteEventTest.java new file mode 100644 index 0000000..e8af081 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/buildtool/TargetCompleteEventTest.java
@@ -0,0 +1,234 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.Iterables; +import com.google.common.eventbus.Subscribe; +import com.google.common.hash.HashCode; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.CompletionContext; +import com.google.devtools.build.lib.actions.FileArtifactValue; +import com.google.devtools.build.lib.actions.FileStateType; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.OutputGroupInfo; +import com.google.devtools.build.lib.analysis.TargetCompleteEvent; +import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.buildeventservice.BazelBuildEventServiceModule; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildEvent; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildEventId.IdCase; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.NamedSetOfFiles; +import com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.packages.util.MockGenruleSupport; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.NoSpawnCacheModule; +import com.google.devtools.build.lib.vfs.DigestHashFunction; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.Nullable; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Verifies TargetCompleteEvent behavior during a complete build. */ +@RunWith(JUnit4.class) +public final class TargetCompleteEventTest extends BuildIntegrationTestCase { + + @Rule public final TemporaryFolder tmpFolder = new TemporaryFolder(); + + @Before + public void stageEmbeddedTools() throws Exception { + AnalysisMock.get().setupMockToolsRepository(mockToolsConfig); + } + + @Override + protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception { + return super.getRuntimeBuilder() + .addBlazeModule(new NoSpawnCacheModule()) + .addBlazeModule(new BazelBuildEventServiceModule()); + } + + private void afterBuildCommand() throws Exception { + runtimeWrapper.newCommand(); + } + + /** + * Validates that TargetCompleteEvents do not keep a map of action output metadata for the + * _validation output group, which can be quite large. + */ + @Test + public void artifactsNotRetained() throws Exception { + write( + "validation_actions/defs.bzl", + "def _rule_with_implicit_outs_and_validation_impl(ctx):", + "", + " ctx.actions.write(ctx.outputs.main, \"main output\\n\")", + "", + " ctx.actions.write(ctx.outputs.implicit, \"implicit output\\n\")", + "", + " validation_output = ctx.actions.declare_file(ctx.attr.name + \".validation\")", + " # The actual tool will be created in individual tests, depending on whether", + " # validation should pass or fail.", + " ctx.actions.run(", + " outputs = [validation_output],", + " executable = ctx.executable._validation_tool,", + " arguments = [validation_output.path])", + "", + " return [", + " DefaultInfo(files = depset([ctx.outputs.main])),", + " OutputGroupInfo(_validation = depset([validation_output])),", + " ]", + "", + "", + "rule_with_implicit_outs_and_validation = rule(", + " implementation = _rule_with_implicit_outs_and_validation_impl,", + " outputs = {", + " \"main\": \"%{name}.main\",", + " \"implicit\": \"%{name}.implicit\",", + " },", + " attrs = {", + " \"_validation_tool\": attr.label(", + " allow_single_file = True,", + " default = Label(\"//validation_actions:validation_tool\"),", + " executable = True,", + " cfg = \"host\"),", + " }", + ")"); + write("validation_actions/validation_tool", "#!/bin/bash", "echo \"validation output\" > $1") + .setExecutable(true); + write( + "validation_actions/BUILD", + "load(", + " \":defs.bzl\",", + " \"rule_with_implicit_outs_and_validation\")", + "", + "rule_with_implicit_outs_and_validation(name = \"foo0\")"); + + AtomicReference<TargetCompleteEvent> targetCompleteEventRef = new AtomicReference<>(); + runtimeWrapper.registerSubscriber( + new Object() { + @SuppressWarnings("unused") + @Subscribe + public void accept(TargetCompleteEvent event) { + targetCompleteEventRef.set(event); + } + }); + + addOptions("--experimental_run_validations"); + BuildResult buildResult = buildTarget("//validation_actions:foo0"); + + Collection<ConfiguredTarget> successfulTargets = buildResult.getSuccessfulTargets(); + ConfiguredTarget fooTarget = Iterables.getOnlyElement(successfulTargets); + + // Check that the primary output, :foo0.main, has its metadata retained and the + // CompletionContext can confirm it is an output file. + Artifact main = + ((RuleConfiguredTarget) fooTarget) + .getArtifactByOutputLabel( + Label.parseAbsoluteUnchecked("//validation_actions:foo0.main")); + FileStateType mainType = + targetCompleteEventRef.get().getCompletionContext().getFileArtifactValue(main).getType(); + assertThat(CompletionContext.isGuaranteedToBeOutputFile(mainType)).isTrue(); + + // Check that the validation output, :foo0.validation, does not have its metadata retained and + // the CompletionContext cannot confirm it is an output file (even though it is). + OutputGroupInfo outputGroups = fooTarget.get(OutputGroupInfo.STARLARK_CONSTRUCTOR); + NestedSet<Artifact> validationArtifacts = + outputGroups.getOutputGroup(OutputGroupInfo.VALIDATION); + assertThat(validationArtifacts.isEmpty()).isFalse(); + + Artifact validationArtifact = Iterables.getOnlyElement(validationArtifacts.toList()); + + FileArtifactValue validationArtifactMetadata = + targetCompleteEventRef + .get() + .getCompletionContext() + .getFileArtifactValue(validationArtifact); + assertThat(validationArtifactMetadata).isNull(); + } + + @Test + public void digestAndLengthInBuildEventProtocol() throws Exception { + MockGenruleSupport.setup(mockToolsConfig); + + // Produces a TargetCompleteEvent in BEP and verifies that we include the output file's + // length and digest. + write( + "foo/BUILD", + "genrule(name = 'foobin', outs = ['out.txt'], cmd = 'echo -n \"Hello\" > $@')"); + File buildEventBinaryFile = tmpFolder.newFile(); + // We use WAIT_FOR_UPLOAD_COMPLETE because it's the easiest way to force the BES module to + // wait until the BEP binary file has been written. + addOptions( + "--build_event_binary_file=" + buildEventBinaryFile.getAbsolutePath(), + "--bes_upload_mode=WAIT_FOR_UPLOAD_COMPLETE"); + buildTarget("//foo:foobin"); + // We need to wait for all events to be written to the file, which is done in #afterCommand() + // if --bes_upload_mode=WAIT_FOR_UPLOAD_COMPLETE. + afterBuildCommand(); + + List<BuildEvent> buildEvents = new ArrayList<>(); + try (InputStream in = new FileInputStream(buildEventBinaryFile)) { + while (in.available() > 0) { + buildEvents.add(BuildEvent.parseDelimitedFrom(in)); + } + } + BuildEventStreamProtos.File outFile = findOutputFileInBEPStream(buildEvents); + assertThat(outFile).isNotNull(); + assertThat(outFile.getLength()).isEqualTo("Hello".length()); + byte[] bepDigest = BaseEncoding.base16().lowerCase().decode(outFile.getDigest()); + // Try all registered hash functions and verify that one of them was used to produce the digest. + boolean foundHashFunction = false; + for (DigestHashFunction hashFunction : DigestHashFunction.getPossibleHashFunctions()) { + HashCode hashCode = + hashFunction.getHashFunction().hashString("Hello", StandardCharsets.UTF_8); + if (Arrays.equals(bepDigest, hashCode.asBytes())) { + foundHashFunction = true; + } + } + assertThat(foundHashFunction).isTrue(); + } + + @Nullable + private static BuildEventStreamProtos.File findOutputFileInBEPStream( + List<BuildEvent> buildEvents) { + for (BuildEvent buildEvent : buildEvents) { + if (buildEvent.getId().getIdCase() == IdCase.NAMED_SET) { + NamedSetOfFiles namedSetOfFiles = buildEvent.getNamedSetOfFiles(); + for (BuildEventStreamProtos.File file : namedSetOfFiles.getFilesList()) { + if (file.getName().contains("out.txt")) { + return file; + } + } + } + } + return null; + } +}