Post a new build event ConvenienceSymlinksIdentifiedEvent.
The event is posted whenever symlinks are created or deleted. However, if user passes in --experimental_convenience_symlink=ignore, then symlinks will not be created or deleted and the BuildEvent will not be posted.
The new flag --experimental_convenience_symlinks_bep_event will determine whether or not the new build event ConvenienceSymlinksIdentified will be posted. By default the value is false, which will be the same functionality as prior to the adding of the new event. If the value is set to true, the build event ConvenienceSymlinksIdentified will be posted with the managed symlinks.
RELNOTES: Post new ConvenienceSymlinksIdentifiedEvent to the BuildEventProtocol when --experimental_convenience_symlinks_bep_event is enabled.
PiperOrigin-RevId: 287584669
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventId.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventId.java
index bc0cb4b..8d72699 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventId.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventId.java
@@ -331,4 +331,13 @@
BuildEventStreamProtos.BuildEventId.BuildMetricsId.getDefaultInstance())
.build());
}
+
+ public static BuildEventId convenienceSymlinksIdentifiedId() {
+ return new BuildEventId(
+ BuildEventStreamProtos.BuildEventId.newBuilder()
+ .setConvenienceSymlinksIdentified(
+ BuildEventStreamProtos.BuildEventId.ConvenienceSymlinksIdentifiedId
+ .getDefaultInstance())
+ .build());
+ }
}
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto
index adfdbbb..c453393 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto
+++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto
@@ -190,6 +190,9 @@
// of the build.
message BuildMetricsId {}
+ // Identifier of an event providing convenience symlinks information.
+ message ConvenienceSymlinksIdentifiedId {}
+
oneof id {
UnknownBuildEventId unknown = 1;
ProgressId progress = 2;
@@ -215,6 +218,7 @@
BuildMetricsId build_metrics = 22;
WorkspaceConfigId workspace = 23;
BuildMetadataId build_metadata = 24;
+ ConvenienceSymlinksIdentifiedId convenience_symlinks_identified = 25;
}
}
@@ -709,6 +713,47 @@
repeated File log = 1;
}
+// Event describing all convenience symlinks (i.e., workspace symlinks) to be
+// created or deleted once the execution phase has begun. Note that this event
+// does not say anything about whether or not the build tool actually executed
+// these filesystem operations; it only says what logical operations should be
+// performed. This event is emitted exactly once per build; if no symlinks are
+// to be modified, the event is still emitted with empty contents.
+message ConvenienceSymlinksIdentified {
+ repeated ConvenienceSymlink convenience_symlinks = 1;
+}
+
+// The message that contains what type of action to perform on a given path and
+// target of a symlink.
+message ConvenienceSymlink {
+ enum Action {
+ UNKNOWN = 0;
+
+ // Indicates a symlink should be created, or overwritten if it already
+ // exists.
+ CREATE = 1;
+
+ // Indicates a symlink should be deleted if it already exists.
+ DELETE = 2;
+ }
+
+ // The path of the symlink to be created or deleted, absolute or relative to
+ // the workspace, creating any directories necessary. If a symlink already
+ // exists at that location, then it should be replaced by a symlink pointing
+ // to the new target.
+ string path = 1;
+
+ // The operation we are performing on the symlink.
+ Action action = 2;
+
+ // If action is CREATE, this is the target path that the symlink should point
+ // to. If the path points underneath the output base, it is relative to the
+ // output base; otherwise it is absolute.
+ //
+ // If action is DELETE, this field is not set.
+ string target = 3;
+}
+
// Message describing a build event. Events will have an identifier that
// is unique within a given build invocation; they also announce follow-up
// events as children. More details, which are specific to the kind of event
@@ -741,5 +786,6 @@
BuildMetrics build_metrics = 24;
WorkspaceConfig workspace_info = 25;
BuildMetadata build_metadata = 26;
+ ConvenienceSymlinksIdentified convenience_symlinks_identified = 27;
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java
index 95c727b..d4338b7 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java
@@ -227,7 +227,8 @@
+ "as determined by the build.\n"
+ " clean: All symlinks will be unconditionally deleted.\n"
+ " ignore: Symlinks will be left alone.\n"
- + " log_only: to be implemented.\n"
+ + " log_only: Generate log messages as if 'normal' were passed, but don't actually "
+ + "perform any filesystem operations (useful for tools).\n"
+ "Note that only symlinks whose names are generated by the current value of "
+ "--symlink_prefix can be affected; if the prefix changes, any pre-existing "
+ "symlinks will be left alone.")
@@ -235,6 +236,19 @@
public ConvenienceSymlinksMode experimentalConvenienceSymlinks;
@Option(
+ name = "experimental_convenience_symlinks_bep_event",
+ defaultValue = "false",
+ documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS,
+ effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+ help =
+ "This flag controls whether or not we will post the build event"
+ + "ConvenienceSymlinksIdentified to the BuildEventProtocol. If the value is true, "
+ + "the BuildEventProtocol will have an entry for convenienceSymlinksIdentified, "
+ + "listing all of the convenience symlinks created in your workspace. If false, then "
+ + "the convenienceSymlinksIdentified entry in the BuildEventProtocol will be empty.")
+ public boolean experimentalConvenienceSymlinksBepEvent;
+
+ @Option(
name = "experimental_multi_cpu",
converter = Converters.CommaSeparatedOptionListConverter.class,
allowMultiple = true,
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
index 3c0a311..9119264 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -47,6 +47,9 @@
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
+import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.ConvenienceSymlink;
+import com.google.devtools.build.lib.buildtool.BuildRequestOptions.ConvenienceSymlinksMode;
+import com.google.devtools.build.lib.buildtool.buildevent.ConvenienceSymlinksIdentifiedEvent;
import com.google.devtools.build.lib.buildtool.buildevent.ExecRootPreparedEvent;
import com.google.devtools.build.lib.buildtool.buildevent.ExecutionPhaseCompleteEvent;
import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent;
@@ -251,7 +254,7 @@
createActionLogDirectory();
}
- createConvenienceSymlinks(request.getBuildOptions(), analysisResult);
+ handleConvenienceSymlinks(analysisResult);
ActionCache actionCache = getActionCache();
actionCache.resetStatistics();
@@ -464,6 +467,23 @@
}
/**
+ * Handles what action to perform on the convenience symlinks. If the the mode is {@link
+ * ConvenienceSymlinksMode.IGNORE}, then skip any creating or cleaning of convenience symlinks.
+ * Otherwise, manage the convenience symlinks and then post a {@link
+ * ConvenienceSymlinksIdentifiedEvent} build event.
+ */
+ private void handleConvenienceSymlinks(AnalysisResult analysisResult) {
+ ImmutableList<ConvenienceSymlink> convenienceSymlinks = ImmutableList.of();
+ if (request.getBuildOptions().experimentalConvenienceSymlinks
+ != ConvenienceSymlinksMode.IGNORE) {
+ convenienceSymlinks = createConvenienceSymlinks(request.getBuildOptions(), analysisResult);
+ }
+ if (request.getBuildOptions().experimentalConvenienceSymlinksBepEvent) {
+ env.getEventBus().post(new ConvenienceSymlinksIdentifiedEvent(convenienceSymlinks));
+ }
+ }
+
+ /**
* Creates convenience symlinks based on the target configurations.
*
* <p>Exactly what target configurations we consider depends on the value of {@code
@@ -476,7 +496,7 @@
* path the symlink should point to, it gets created; otherwise, the symlink is not created, and
* in fact gets removed if it was already present from a previous invocation.
*/
- private void createConvenienceSymlinks(
+ private ImmutableList<ConvenienceSymlink> createConvenienceSymlinks(
BuildRequestOptions buildRequestOptions, AnalysisResult analysisResult) {
SkyframeExecutor executor = env.getSkyframeExecutor();
Reporter reporter = env.getReporter();
@@ -494,16 +514,14 @@
analysisResult.getConfigurationCollection().getTargetConfigurations());
String productName = runtime.getProductName();
- String workspaceName = env.getWorkspaceName();
try (SilentCloseable c =
Profiler.instance().profile("OutputDirectoryLinksUtils.createOutputDirectoryLinks")) {
- OutputDirectoryLinksUtils.createOutputDirectoryLinks(
+ return OutputDirectoryLinksUtils.createOutputDirectoryLinks(
runtime.getRuleClassProvider().getSymlinkDefinitions(),
buildRequestOptions,
- workspaceName,
+ env.getWorkspaceName(),
env.getWorkspace(),
- env.getDirectories().getExecRoot(workspaceName),
- env.getDirectories().getOutputPath(workspaceName),
+ env.getDirectories(),
getReporter(),
targetConfigurations,
options -> getConfiguration(executor, reporter, options),
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java
index 32f0ae9..a8fba52 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java
@@ -17,11 +17,14 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.ConvenienceSymlinks;
import com.google.devtools.build.lib.analysis.config.ConvenienceSymlinks.OutputSymlink;
import com.google.devtools.build.lib.analysis.config.ConvenienceSymlinks.SymlinkDefinition;
+import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.ConvenienceSymlink;
+import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.ConvenienceSymlink.Action;
import com.google.devtools.build.lib.buildtool.BuildRequestOptions.ConvenienceSymlinksMode;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.events.Event;
@@ -93,29 +96,36 @@
*
* <p>A warning is emitted if a symlink would resolve to multiple destinations, or if a filesystem
* mutation operation fails.
+ *
+ * @return a list of {@link ConvenienceSymlink} messages describing what was created and
+ * destroyed.
*/
- static void createOutputDirectoryLinks(
+ static ImmutableList<ConvenienceSymlink> createOutputDirectoryLinks(
Iterable<SymlinkDefinition> symlinkDefinitions,
BuildRequestOptions buildRequestOptions,
String workspaceName,
Path workspace,
- Path execRoot,
- Path outputPath,
+ BlazeDirectories directories,
EventHandler eventHandler,
Set<BuildConfiguration> targetConfigs,
Function<BuildOptions, BuildConfiguration> configGetter,
String productName) {
+ Path execRoot = directories.getExecRoot(workspaceName);
+ Path outputPath = directories.getOutputPath(workspaceName);
+ Path outputBase = directories.getOutputBase();
String symlinkPrefix = buildRequestOptions.getSymlinkPrefix(productName);
ConvenienceSymlinksMode mode = buildRequestOptions.experimentalConvenienceSymlinks;
- if (mode == ConvenienceSymlinksMode.IGNORE || NO_CREATE_SYMLINKS_PREFIX.equals(symlinkPrefix)) {
- return;
+ if (NO_CREATE_SYMLINKS_PREFIX.equals(symlinkPrefix)) {
+ return ImmutableList.of();
}
+ ImmutableList.Builder<ConvenienceSymlink> convenienceSymlinksBuilder = ImmutableList.builder();
List<String> failures = new ArrayList<>();
List<String> ambiguousLinks = new ArrayList<>();
Set<String> createdLinks = new LinkedHashSet<>();
String workspaceBaseName = workspace.getBaseName();
RepositoryName repositoryName = RepositoryName.createFromValidStrippedName(workspaceName);
+ boolean logOnly = mode == ConvenienceSymlinksMode.LOG_ONLY;
for (SymlinkDefinition symlink : getAllLinkDefinitions(symlinkDefinitions)) {
String linkName = symlink.getLinkName(symlinkPrefix, productName, workspaceBaseName);
@@ -124,7 +134,7 @@
continue;
}
if (mode == ConvenienceSymlinksMode.CLEAN) {
- removeLink(workspace, linkName, failures);
+ removeLink(workspace, linkName, failures, convenienceSymlinksBuilder, logOnly);
} else {
Set<Path> candidatePaths =
symlink.getLinkPaths(
@@ -135,9 +145,16 @@
outputPath,
execRoot);
if (candidatePaths.size() == 1) {
- createLink(workspace, linkName, Iterables.getOnlyElement(candidatePaths), failures);
+ createLink(
+ workspace,
+ linkName,
+ outputBase,
+ Iterables.getOnlyElement(candidatePaths),
+ failures,
+ convenienceSymlinksBuilder,
+ logOnly);
} else {
- removeLink(workspace, linkName, failures);
+ removeLink(workspace, linkName, failures, convenienceSymlinksBuilder, logOnly);
// candidatePaths can be empty if the symlink decided not to be created. This can happen
// if the symlink is disabled by a flag, or it intercepts an error while computing its
// target path. In that case, don't trigger a warning about an ambiguous link.
@@ -160,6 +177,7 @@
"cleared convenience symlink(s) %s because their destinations would be ambiguous",
Joiner.on(", ").join(ambiguousLinks))));
}
+ return convenienceSymlinksBuilder.build();
}
public static PathPrettyPrinter getPathPrettyPrinter(
@@ -203,7 +221,11 @@
String workspaceBaseName = workspace.getBaseName();
for (SymlinkDefinition link : getAllLinkDefinitions(symlinkDefinitions)) {
removeLink(
- workspace, link.getLinkName(symlinkPrefix, productName, workspaceBaseName), failures);
+ workspace,
+ link.getLinkName(symlinkPrefix, productName, workspaceBaseName),
+ failures,
+ ImmutableList.builder(),
+ false);
}
FileSystemUtils.removeDirectoryAndParents(workspace, PathFragment.create(symlinkPrefix));
@@ -215,9 +237,44 @@
}
/**
- * Helper to createOutputDirectoryLinks that creates a symlink from base + name to target.
+ * Creates a symlink and outputs a {@link ConvenienceSymlink} entry.
+ *
+ * <p>The symlink is created at path {@code name}, relative to {@code base}, creating directories
+ * as needed; it points to {@code target}. Any filesystem errors are appended to {@code failures}.
+ *
+ * <p>A {@code ConvenienceSymlink} entry is added to {@code symlinksBuilder} describing the
+ * symlink. {@code outputBase} is used to determine the relative target path for this entry.
+ *
+ * <p>If {@code logOnly} is true, the {@code ConvenienceSymlink} entry is added but no actual
+ * filesystem operations are performed.
+ *
+ * @return true iff there were no filesystem errors.
*/
- private static boolean createLink(Path base, String name, Path target, List<String> failures) {
+ private static boolean createLink(
+ Path base,
+ String name,
+ Path outputBase,
+ Path target,
+ List<String> failures,
+ ImmutableList.Builder<ConvenienceSymlink> symlinksBuilder,
+ boolean logOnly) {
+ // Usually the symlink target falls under the output base, and the path in the BEP event should
+ // be relative to that output base. In rare cases where the symlink points elsewhere, use the
+ // absolute path as a fallback.
+ String targetForEvent =
+ target.startsWith(outputBase)
+ ? target.relativeTo(outputBase).getPathString()
+ : target.getPathString();
+ symlinksBuilder.add(
+ ConvenienceSymlink.newBuilder()
+ .setPath(name)
+ .setTarget(targetForEvent)
+ .setAction(Action.CREATE)
+ .build());
+ if (logOnly) {
+ return true;
+ }
+ Path link = base.getRelative(name);
try {
FileSystemUtils.createDirectoryAndParents(target);
} catch (IOException e) {
@@ -226,7 +283,7 @@
return false;
}
try {
- FileSystemUtils.ensureSymbolicLink(base.getRelative(name), target);
+ FileSystemUtils.ensureSymbolicLink(link, target);
} catch (IOException e) {
failures.add(String.format("cannot create symbolic link %s -> %s: %s",
name, target.getPathString(), e.getMessage()));
@@ -237,12 +294,36 @@
}
/**
- * Helper to removeOutputDirectoryLinks that removes one of the Blaze convenience symbolic links.
+ * Deletes a symlink and outputs a {@link ConvenienceSymlink} entry.
+ *
+ * <p>The symlink to be deleted is at path {@code name}, relative to {@code base}. Any filesystem
+ * errors are appended to {@code failures}.
+ *
+ * <p>A {@code ConvenienceSymlink} entry is added to {@code symlinksBuilder} describing the
+ * symlink to be deleted.
+ *
+ * <p>If {@code logOnly} is true, the {@code ConvenienceSymlink} entry is added but no actual
+ * filesystem operations are performed.
+ *
+ * @return true iff there were no filesystem errors.
*/
- private static boolean removeLink(Path base, String name, List<String> failures) {
+ private static boolean removeLink(
+ Path base,
+ String name,
+ List<String> failures,
+ ImmutableList.Builder<ConvenienceSymlink> symlinksBuilder,
+ boolean logOnly) {
+ symlinksBuilder.add(
+ ConvenienceSymlink.newBuilder().setPath(name).setAction(Action.DELETE).build());
+ if (logOnly) {
+ return true;
+ }
Path link = base.getRelative(name);
try {
if (link.isSymbolicLink()) {
+ // TODO(b/146885821): Consider also removing empty ancestor directories, to allow for
+ // cleaning up directories generated by --symlink_prefix=dir1/dir2/...
+ // Might be undesireable since it could also remove manually-created directories.
ExecutionTool.logger.finest("Removing " + link);
link.delete();
}
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ConvenienceSymlinksIdentifiedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ConvenienceSymlinksIdentifiedEvent.java
new file mode 100644
index 0000000..66ef8a2
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ConvenienceSymlinksIdentifiedEvent.java
@@ -0,0 +1,60 @@
+// Copyright 2019 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.buildtool.buildevent;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.buildeventstream.BuildEvent;
+import com.google.devtools.build.lib.buildeventstream.BuildEventContext;
+import com.google.devtools.build.lib.buildeventstream.BuildEventId;
+import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos;
+import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.ConvenienceSymlink;
+import com.google.devtools.build.lib.buildeventstream.GenericBuildEvent;
+import java.util.Collection;
+
+/**
+ * This event is fired from ExecutionTool#handleConvenienceSymlinks() whenever convenience symlinks
+ * are managed. If the value {@link ConvenienceSymlinksMode.NORMAL}, LOG_ONLY, CLEAN is passed into
+ * the build request option {@code --experimental_create_convenience_symlinks}, then this event will
+ * be populated with convenience symlink entries. However, if {@link ConvenienceSymlinksMode.IGNORE}
+ * is passed, then this will be an empty event.
+ */
+public final class ConvenienceSymlinksIdentifiedEvent implements BuildEvent {
+ private final ImmutableList<ConvenienceSymlink> convenienceSymlinks;
+
+ /** Construct the ConvenienceSymlinksIdentifiedEvent. */
+ public ConvenienceSymlinksIdentifiedEvent(ImmutableList<ConvenienceSymlink> convenienceSymlinks) {
+ this.convenienceSymlinks = convenienceSymlinks;
+ }
+
+ @Override
+ public BuildEventId getEventId() {
+ return BuildEventId.convenienceSymlinksIdentifiedId();
+ }
+
+ @Override
+ public Collection<BuildEventId> getChildrenEvents() {
+ return ImmutableList.of();
+ }
+
+ @Override
+ public BuildEventStreamProtos.BuildEvent asStreamProto(BuildEventContext converters) {
+ BuildEventStreamProtos.ConvenienceSymlinksIdentified convenienceSymlinksIdentified =
+ BuildEventStreamProtos.ConvenienceSymlinksIdentified.newBuilder()
+ .addAllConvenienceSymlinks(convenienceSymlinks)
+ .build();
+ return GenericBuildEvent.protoChaining(this)
+ .setConvenienceSymlinksIdentified(convenienceSymlinksIdentified)
+ .build();
+ }
+}
