blob: d9ae7c2bf8bbf29f2f0aa89352a049d28a9b4dce [file] [log] [blame]
// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.analysis;
import static java.nio.charset.StandardCharsets.ISO_8859_1;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionKeyContext;
import com.google.devtools.build.lib.actions.ActionOwner;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec.VisibleForSerialization;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
/**
* Creates a manifest file describing a symlink tree.
*
* <p>In addition to symlink trees (whose manifests are a tree position -> exec path map), this
* action can also create manifest consisting of just exec paths for historical reasons.
*
* <p>This action carefully avoids building the manifest content in memory because it can be large.
*/
@AutoCodec
@Immutable // if all ManifestWriter implementations are immutable
public final class SourceManifestAction extends AbstractFileWriteAction {
private static final String GUID = "07459553-a3d0-4d37-9d78-18ed942470f4";
/**
* Interface for defining manifest formatting and reporting specifics. Implementations must be
* immutable.
*/
@VisibleForTesting
interface ManifestWriter {
/**
* Writes a single line of manifest output.
*
* @param manifestWriter the output stream
* @param rootRelativePath path of an entry relative to the manifest's root
* @param symlink (optional) symlink that resolves the above path
*/
void writeEntry(Writer manifestWriter, PathFragment rootRelativePath,
@Nullable Artifact symlink) throws IOException;
/**
* Fulfills {@link com.google.devtools.build.lib.actions.AbstractAction#getMnemonic()}
*/
String getMnemonic();
/**
* Fulfills {@link com.google.devtools.build.lib.actions.AbstractAction#getRawProgressMessage()}
*/
String getRawProgressMessage();
/**
* Fulfills {@link AbstractFileWriteAction#isRemotable()}.
* @return
*/
boolean isRemotable();
}
/**
* The strategy we use to write manifest entries.
*/
private final ManifestWriter manifestWriter;
/**
* The runfiles for which to create the symlink tree.
*/
private final Runfiles runfiles;
/**
* Creates a new AbstractSourceManifestAction instance using latin1 encoding to write the manifest
* file and with a specified root path for manifest entries.
*
* @param manifestWriter the strategy to use to write manifest entries
* @param owner the action owner
* @param primaryOutput the file to which to write the manifest
* @param runfiles runfiles
*/
@VisibleForSerialization
SourceManifestAction(
ManifestWriter manifestWriter, ActionOwner owner, Artifact primaryOutput, Runfiles runfiles) {
super(owner, getDependencies(runfiles), primaryOutput, false);
this.manifestWriter = manifestWriter;
this.runfiles = runfiles;
}
/** The {@link Runfiles} for which this action creates the symlink tree. */
public Runfiles getGeneratedRunfiles() {
return runfiles;
}
@VisibleForTesting
public void writeOutputFile(OutputStream out, EventHandler eventHandler)
throws IOException {
writeFile(out,
runfiles.getRunfilesInputs(
eventHandler,
getOwner().getLocation(),
ArtifactPathResolver.IDENTITY));
}
@Override
public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx)
throws IOException {
final Map<PathFragment, Artifact> runfilesInputs =
runfiles.getRunfilesInputs(ctx.getEventHandler(), getOwner().getLocation(),
ctx.getPathResolver());
return out -> writeFile(out, runfilesInputs);
}
@Override
public boolean isRemotable() {
return manifestWriter.isRemotable();
}
/**
* Returns the input dependencies for this action. Note we don't need to create the symlink
* target Artifacts before we write the output manifest, so this Action does not have to
* depend on them. The only necessary dependencies are pruning manifests, which must be read
* to properly prune the tree.
*/
public static Collection<Artifact> getDependencies(Runfiles runfiles) {
ImmutableList.Builder<Artifact> builder = ImmutableList.builder();
for (Runfiles.PruningManifest manifest : runfiles.getPruningManifests()) {
builder.add(manifest.getManifestFile());
}
return builder.build();
}
/**
* Sort the entries in both the normal and root manifests and write the output
* file.
*
* @param out is the message stream to write errors to.
* @param output The actual mapping of the output manifest.
* @throws IOException
*/
private void writeFile(OutputStream out, Map<PathFragment, Artifact> output) throws IOException {
Writer manifestFile = new BufferedWriter(new OutputStreamWriter(out, ISO_8859_1));
Comparator<Map.Entry<PathFragment, Artifact>> fragmentComparator =
new Comparator<Map.Entry<PathFragment, Artifact>>() {
@Override
public int compare(Map.Entry<PathFragment, Artifact> path1,
Map.Entry<PathFragment, Artifact> path2) {
return path1.getKey().compareTo(path2.getKey());
}
};
List<Map.Entry<PathFragment, Artifact>> sortedManifest = new ArrayList<>(output.entrySet());
Collections.sort(sortedManifest, fragmentComparator);
for (Map.Entry<PathFragment, Artifact> line : sortedManifest) {
manifestWriter.writeEntry(manifestFile, line.getKey(), line.getValue());
}
manifestFile.flush();
}
@Override
public String getMnemonic() {
return manifestWriter.getMnemonic();
}
@Override
protected String getRawProgressMessage() {
return manifestWriter.getRawProgressMessage() + " for " + getOwner().getLabel();
}
@Override
protected void computeKey(ActionKeyContext actionKeyContext, Fingerprint fp) {
fp.addString(GUID);
runfiles.fingerprint(fp);
}
/**
* Supported manifest writing strategies.
*/
public enum ManifestType implements ManifestWriter {
/**
* Writes each line as:
*
* [rootRelativePath] [resolvingSymlink]
*
* <p>This strategy is suitable for creating an input manifest to a source view tree. Its
* output is a valid input to
* {@link com.google.devtools.build.lib.analysis.actions.SymlinkTreeAction}.
*/
SOURCE_SYMLINKS {
@Override
public void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, Artifact symlink)
throws IOException {
manifestWriter.append(rootRelativePath.getPathString());
// This trailing whitespace is REQUIRED to process the single entry line correctly.
manifestWriter.append(' ');
if (symlink != null) {
manifestWriter.append(symlink.getPath().getPathString());
}
manifestWriter.append('\n');
}
@Override
public String getMnemonic() {
return "SourceSymlinkManifest";
}
@Override
public String getRawProgressMessage() {
return "Creating source manifest";
}
@Override
public boolean isRemotable() {
// There is little gain to remoting these, since they include absolute path names inline.
return false;
}
},
/**
* Writes each line as:
*
* [rootRelativePath]
*
* <p>This strategy is suitable for an input into a packaging system (notably .par) that
* consumes a list of all source files but needs that list to be constant with respect to
* how the user has their client laid out on local disk.
*/
SOURCES_ONLY {
@Override
public void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, Artifact symlink)
throws IOException {
manifestWriter.append(rootRelativePath.getPathString());
manifestWriter.append('\n');
manifestWriter.flush();
}
@Override
public String getMnemonic() {
return "PackagingSourcesManifest";
}
@Override
public String getRawProgressMessage() {
return "Creating file sources list";
}
@Override
public boolean isRemotable() {
// Source-only symlink manifest has root-relative paths and does not include absolute paths.
return true;
}
}
}
/** Creates an action for the given runfiles. */
public static SourceManifestAction forRunfiles(ManifestType manifestType, ActionOwner owner,
Artifact output, Runfiles runfiles) {
return new SourceManifestAction(manifestType, owner, output, runfiles);
}
/**
* Builder class to construct {@link SourceManifestAction} instances.
*/
public static final class Builder {
private final ManifestWriter manifestWriter;
private final ActionOwner owner;
private final Artifact output;
private final Runfiles.Builder runfilesBuilder;
public Builder(String prefix, ManifestType manifestType, ActionOwner owner, Artifact output,
boolean legacyExternalRunfiles) {
this(manifestType, owner, output, new Runfiles.Builder(prefix, legacyExternalRunfiles));
}
public Builder(
ManifestType manifestType,
ActionOwner owner,
Artifact output,
Runfiles.Builder runfilesBuilder) {
this.manifestWriter = manifestType;
this.owner = owner;
this.output = output;
this.runfilesBuilder = runfilesBuilder;
}
@VisibleForTesting // Only used for testing.
Builder(String prefix, ManifestWriter manifestWriter, ActionOwner owner, Artifact output) {
this.runfilesBuilder = new Runfiles.Builder(prefix, false);
this.manifestWriter = manifestWriter;
this.owner = owner;
this.output = output;
}
public SourceManifestAction build() {
return new SourceManifestAction(manifestWriter, owner, output, runfilesBuilder.build());
}
/**
* Adds a set of symlinks from the artifacts' root-relative paths to the
* artifacts themselves.
*/
public Builder addSymlinks(Iterable<Artifact> artifacts) {
runfilesBuilder.addArtifacts(artifacts);
return this;
}
/**
* Adds a map of symlinks.
*/
public Builder addSymlinks(Map<PathFragment, Artifact> symlinks) {
runfilesBuilder.addSymlinks(symlinks);
return this;
}
/**
* Adds a single symlink.
*/
public Builder addSymlink(PathFragment link, Artifact target) {
runfilesBuilder.addSymlink(link, target);
return this;
}
/**
* <p>Adds a mapping of Artifacts to the directory above the normal symlink
* forest base.
*/
public Builder addRootSymlinks(Map<PathFragment, Artifact> rootSymlinks) {
runfilesBuilder.addRootSymlinks(rootSymlinks);
return this;
}
/**
* Set the empty files supplier for the manifest, see {@link Runfiles.EmptyFilesSupplier}
* for more details.
*/
public Builder setEmptyFilesSupplier(Runfiles.EmptyFilesSupplier supplier) {
runfilesBuilder.setEmptyFilesSupplier(supplier);
return this;
}
/**
* Adds a runfiles pruning manifest.
*/
@VisibleForTesting
Builder addPruningManifest(Runfiles.PruningManifest manifest) {
runfilesBuilder.addPruningManifest(manifest);
return this;
}
}
}