Post a new build event ConvenienceSymlinksIdentifiedEvent. The event is posted whenever symlinks are created or deleted. However, if user passes in --experimental_convenience_symlink=ignore, then symlinks will not be created or deleted and the BuildEvent will not be posted. The new flag --experimental_convenience_symlinks_bep_event will determine whether or not the new build event ConvenienceSymlinksIdentified will be posted. By default the value is false, which will be the same functionality as prior to the adding of the new event. If the value is set to true, the build event ConvenienceSymlinksIdentified will be posted with the managed symlinks. RELNOTES: Post new ConvenienceSymlinksIdentifiedEvent to the BuildEventProtocol when --experimental_convenience_symlinks_bep_event is enabled. PiperOrigin-RevId: 287584669
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java index 95c727b..d4338b7 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java
@@ -227,7 +227,8 @@ + "as determined by the build.\n" + " clean: All symlinks will be unconditionally deleted.\n" + " ignore: Symlinks will be left alone.\n" - + " log_only: to be implemented.\n" + + " log_only: Generate log messages as if 'normal' were passed, but don't actually " + + "perform any filesystem operations (useful for tools).\n" + "Note that only symlinks whose names are generated by the current value of " + "--symlink_prefix can be affected; if the prefix changes, any pre-existing " + "symlinks will be left alone.") @@ -235,6 +236,19 @@ public ConvenienceSymlinksMode experimentalConvenienceSymlinks; @Option( + name = "experimental_convenience_symlinks_bep_event", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = + "This flag controls whether or not we will post the build event" + + "ConvenienceSymlinksIdentified to the BuildEventProtocol. If the value is true, " + + "the BuildEventProtocol will have an entry for convenienceSymlinksIdentified, " + + "listing all of the convenience symlinks created in your workspace. If false, then " + + "the convenienceSymlinksIdentified entry in the BuildEventProtocol will be empty.") + public boolean experimentalConvenienceSymlinksBepEvent; + + @Option( name = "experimental_multi_cpu", converter = Converters.CommaSeparatedOptionListConverter.class, allowMultiple = true,
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index 3c0a311..9119264 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -47,6 +47,9 @@ import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.ConvenienceSymlink; +import com.google.devtools.build.lib.buildtool.BuildRequestOptions.ConvenienceSymlinksMode; +import com.google.devtools.build.lib.buildtool.buildevent.ConvenienceSymlinksIdentifiedEvent; import com.google.devtools.build.lib.buildtool.buildevent.ExecRootPreparedEvent; import com.google.devtools.build.lib.buildtool.buildevent.ExecutionPhaseCompleteEvent; import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent; @@ -251,7 +254,7 @@ createActionLogDirectory(); } - createConvenienceSymlinks(request.getBuildOptions(), analysisResult); + handleConvenienceSymlinks(analysisResult); ActionCache actionCache = getActionCache(); actionCache.resetStatistics(); @@ -464,6 +467,23 @@ } /** + * Handles what action to perform on the convenience symlinks. If the the mode is {@link + * ConvenienceSymlinksMode.IGNORE}, then skip any creating or cleaning of convenience symlinks. + * Otherwise, manage the convenience symlinks and then post a {@link + * ConvenienceSymlinksIdentifiedEvent} build event. + */ + private void handleConvenienceSymlinks(AnalysisResult analysisResult) { + ImmutableList<ConvenienceSymlink> convenienceSymlinks = ImmutableList.of(); + if (request.getBuildOptions().experimentalConvenienceSymlinks + != ConvenienceSymlinksMode.IGNORE) { + convenienceSymlinks = createConvenienceSymlinks(request.getBuildOptions(), analysisResult); + } + if (request.getBuildOptions().experimentalConvenienceSymlinksBepEvent) { + env.getEventBus().post(new ConvenienceSymlinksIdentifiedEvent(convenienceSymlinks)); + } + } + + /** * Creates convenience symlinks based on the target configurations. * * <p>Exactly what target configurations we consider depends on the value of {@code @@ -476,7 +496,7 @@ * path the symlink should point to, it gets created; otherwise, the symlink is not created, and * in fact gets removed if it was already present from a previous invocation. */ - private void createConvenienceSymlinks( + private ImmutableList<ConvenienceSymlink> createConvenienceSymlinks( BuildRequestOptions buildRequestOptions, AnalysisResult analysisResult) { SkyframeExecutor executor = env.getSkyframeExecutor(); Reporter reporter = env.getReporter(); @@ -494,16 +514,14 @@ analysisResult.getConfigurationCollection().getTargetConfigurations()); String productName = runtime.getProductName(); - String workspaceName = env.getWorkspaceName(); try (SilentCloseable c = Profiler.instance().profile("OutputDirectoryLinksUtils.createOutputDirectoryLinks")) { - OutputDirectoryLinksUtils.createOutputDirectoryLinks( + return OutputDirectoryLinksUtils.createOutputDirectoryLinks( runtime.getRuleClassProvider().getSymlinkDefinitions(), buildRequestOptions, - workspaceName, + env.getWorkspaceName(), env.getWorkspace(), - env.getDirectories().getExecRoot(workspaceName), - env.getDirectories().getOutputPath(workspaceName), + env.getDirectories(), getReporter(), targetConfigurations, options -> getConfiguration(executor, reporter, options),
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java index 32f0ae9..a8fba52 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java
@@ -17,11 +17,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.config.ConvenienceSymlinks; import com.google.devtools.build.lib.analysis.config.ConvenienceSymlinks.OutputSymlink; import com.google.devtools.build.lib.analysis.config.ConvenienceSymlinks.SymlinkDefinition; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.ConvenienceSymlink; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.ConvenienceSymlink.Action; import com.google.devtools.build.lib.buildtool.BuildRequestOptions.ConvenienceSymlinksMode; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; @@ -93,29 +96,36 @@ * * <p>A warning is emitted if a symlink would resolve to multiple destinations, or if a filesystem * mutation operation fails. + * + * @return a list of {@link ConvenienceSymlink} messages describing what was created and + * destroyed. */ - static void createOutputDirectoryLinks( + static ImmutableList<ConvenienceSymlink> createOutputDirectoryLinks( Iterable<SymlinkDefinition> symlinkDefinitions, BuildRequestOptions buildRequestOptions, String workspaceName, Path workspace, - Path execRoot, - Path outputPath, + BlazeDirectories directories, EventHandler eventHandler, Set<BuildConfiguration> targetConfigs, Function<BuildOptions, BuildConfiguration> configGetter, String productName) { + Path execRoot = directories.getExecRoot(workspaceName); + Path outputPath = directories.getOutputPath(workspaceName); + Path outputBase = directories.getOutputBase(); String symlinkPrefix = buildRequestOptions.getSymlinkPrefix(productName); ConvenienceSymlinksMode mode = buildRequestOptions.experimentalConvenienceSymlinks; - if (mode == ConvenienceSymlinksMode.IGNORE || NO_CREATE_SYMLINKS_PREFIX.equals(symlinkPrefix)) { - return; + if (NO_CREATE_SYMLINKS_PREFIX.equals(symlinkPrefix)) { + return ImmutableList.of(); } + ImmutableList.Builder<ConvenienceSymlink> convenienceSymlinksBuilder = ImmutableList.builder(); List<String> failures = new ArrayList<>(); List<String> ambiguousLinks = new ArrayList<>(); Set<String> createdLinks = new LinkedHashSet<>(); String workspaceBaseName = workspace.getBaseName(); RepositoryName repositoryName = RepositoryName.createFromValidStrippedName(workspaceName); + boolean logOnly = mode == ConvenienceSymlinksMode.LOG_ONLY; for (SymlinkDefinition symlink : getAllLinkDefinitions(symlinkDefinitions)) { String linkName = symlink.getLinkName(symlinkPrefix, productName, workspaceBaseName); @@ -124,7 +134,7 @@ continue; } if (mode == ConvenienceSymlinksMode.CLEAN) { - removeLink(workspace, linkName, failures); + removeLink(workspace, linkName, failures, convenienceSymlinksBuilder, logOnly); } else { Set<Path> candidatePaths = symlink.getLinkPaths( @@ -135,9 +145,16 @@ outputPath, execRoot); if (candidatePaths.size() == 1) { - createLink(workspace, linkName, Iterables.getOnlyElement(candidatePaths), failures); + createLink( + workspace, + linkName, + outputBase, + Iterables.getOnlyElement(candidatePaths), + failures, + convenienceSymlinksBuilder, + logOnly); } else { - removeLink(workspace, linkName, failures); + removeLink(workspace, linkName, failures, convenienceSymlinksBuilder, logOnly); // candidatePaths can be empty if the symlink decided not to be created. This can happen // if the symlink is disabled by a flag, or it intercepts an error while computing its // target path. In that case, don't trigger a warning about an ambiguous link. @@ -160,6 +177,7 @@ "cleared convenience symlink(s) %s because their destinations would be ambiguous", Joiner.on(", ").join(ambiguousLinks)))); } + return convenienceSymlinksBuilder.build(); } public static PathPrettyPrinter getPathPrettyPrinter( @@ -203,7 +221,11 @@ String workspaceBaseName = workspace.getBaseName(); for (SymlinkDefinition link : getAllLinkDefinitions(symlinkDefinitions)) { removeLink( - workspace, link.getLinkName(symlinkPrefix, productName, workspaceBaseName), failures); + workspace, + link.getLinkName(symlinkPrefix, productName, workspaceBaseName), + failures, + ImmutableList.builder(), + false); } FileSystemUtils.removeDirectoryAndParents(workspace, PathFragment.create(symlinkPrefix)); @@ -215,9 +237,44 @@ } /** - * Helper to createOutputDirectoryLinks that creates a symlink from base + name to target. + * Creates a symlink and outputs a {@link ConvenienceSymlink} entry. + * + * <p>The symlink is created at path {@code name}, relative to {@code base}, creating directories + * as needed; it points to {@code target}. Any filesystem errors are appended to {@code failures}. + * + * <p>A {@code ConvenienceSymlink} entry is added to {@code symlinksBuilder} describing the + * symlink. {@code outputBase} is used to determine the relative target path for this entry. + * + * <p>If {@code logOnly} is true, the {@code ConvenienceSymlink} entry is added but no actual + * filesystem operations are performed. + * + * @return true iff there were no filesystem errors. */ - private static boolean createLink(Path base, String name, Path target, List<String> failures) { + private static boolean createLink( + Path base, + String name, + Path outputBase, + Path target, + List<String> failures, + ImmutableList.Builder<ConvenienceSymlink> symlinksBuilder, + boolean logOnly) { + // Usually the symlink target falls under the output base, and the path in the BEP event should + // be relative to that output base. In rare cases where the symlink points elsewhere, use the + // absolute path as a fallback. + String targetForEvent = + target.startsWith(outputBase) + ? target.relativeTo(outputBase).getPathString() + : target.getPathString(); + symlinksBuilder.add( + ConvenienceSymlink.newBuilder() + .setPath(name) + .setTarget(targetForEvent) + .setAction(Action.CREATE) + .build()); + if (logOnly) { + return true; + } + Path link = base.getRelative(name); try { FileSystemUtils.createDirectoryAndParents(target); } catch (IOException e) { @@ -226,7 +283,7 @@ return false; } try { - FileSystemUtils.ensureSymbolicLink(base.getRelative(name), target); + FileSystemUtils.ensureSymbolicLink(link, target); } catch (IOException e) { failures.add(String.format("cannot create symbolic link %s -> %s: %s", name, target.getPathString(), e.getMessage())); @@ -237,12 +294,36 @@ } /** - * Helper to removeOutputDirectoryLinks that removes one of the Blaze convenience symbolic links. + * Deletes a symlink and outputs a {@link ConvenienceSymlink} entry. + * + * <p>The symlink to be deleted is at path {@code name}, relative to {@code base}. Any filesystem + * errors are appended to {@code failures}. + * + * <p>A {@code ConvenienceSymlink} entry is added to {@code symlinksBuilder} describing the + * symlink to be deleted. + * + * <p>If {@code logOnly} is true, the {@code ConvenienceSymlink} entry is added but no actual + * filesystem operations are performed. + * + * @return true iff there were no filesystem errors. */ - private static boolean removeLink(Path base, String name, List<String> failures) { + private static boolean removeLink( + Path base, + String name, + List<String> failures, + ImmutableList.Builder<ConvenienceSymlink> symlinksBuilder, + boolean logOnly) { + symlinksBuilder.add( + ConvenienceSymlink.newBuilder().setPath(name).setAction(Action.DELETE).build()); + if (logOnly) { + return true; + } Path link = base.getRelative(name); try { if (link.isSymbolicLink()) { + // TODO(b/146885821): Consider also removing empty ancestor directories, to allow for + // cleaning up directories generated by --symlink_prefix=dir1/dir2/... + // Might be undesireable since it could also remove manually-created directories. ExecutionTool.logger.finest("Removing " + link); link.delete(); }
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ConvenienceSymlinksIdentifiedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ConvenienceSymlinksIdentifiedEvent.java new file mode 100644 index 0000000..66ef8a2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ConvenienceSymlinksIdentifiedEvent.java
@@ -0,0 +1,60 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool.buildevent; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.buildeventstream.BuildEvent; +import com.google.devtools.build.lib.buildeventstream.BuildEventContext; +import com.google.devtools.build.lib.buildeventstream.BuildEventId; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.ConvenienceSymlink; +import com.google.devtools.build.lib.buildeventstream.GenericBuildEvent; +import java.util.Collection; + +/** + * This event is fired from ExecutionTool#handleConvenienceSymlinks() whenever convenience symlinks + * are managed. If the value {@link ConvenienceSymlinksMode.NORMAL}, LOG_ONLY, CLEAN is passed into + * the build request option {@code --experimental_create_convenience_symlinks}, then this event will + * be populated with convenience symlink entries. However, if {@link ConvenienceSymlinksMode.IGNORE} + * is passed, then this will be an empty event. + */ +public final class ConvenienceSymlinksIdentifiedEvent implements BuildEvent { + private final ImmutableList<ConvenienceSymlink> convenienceSymlinks; + + /** Construct the ConvenienceSymlinksIdentifiedEvent. */ + public ConvenienceSymlinksIdentifiedEvent(ImmutableList<ConvenienceSymlink> convenienceSymlinks) { + this.convenienceSymlinks = convenienceSymlinks; + } + + @Override + public BuildEventId getEventId() { + return BuildEventId.convenienceSymlinksIdentifiedId(); + } + + @Override + public Collection<BuildEventId> getChildrenEvents() { + return ImmutableList.of(); + } + + @Override + public BuildEventStreamProtos.BuildEvent asStreamProto(BuildEventContext converters) { + BuildEventStreamProtos.ConvenienceSymlinksIdentified convenienceSymlinksIdentified = + BuildEventStreamProtos.ConvenienceSymlinksIdentified.newBuilder() + .addAllConvenienceSymlinks(convenienceSymlinks) + .build(); + return GenericBuildEvent.protoChaining(this) + .setConvenienceSymlinksIdentified(convenienceSymlinksIdentified) + .build(); + } +}