src/main/java/com/google/devtools/build/lib/analysis/SourceManifestAction.java

// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.analysis;

import static java.nio.charset.StandardCharsets.ISO_8859_1;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionKeyContext;
import com.google.devtools.build.lib.actions.ActionOwner;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactPathResolver;
import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec.VisibleForSerialization;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;

/**
 * Creates a manifest file describing a symlink tree.
 *
 * <p>In addition to symlink trees (whose manifests are a tree position -> exec path map), this
 * action can also create manifest consisting of just exec paths for historical reasons.
 *
 * <p>This action carefully avoids building the manifest content in memory because it can be large.
 */
@AutoCodec
@Immutable // if all ManifestWriter implementations are immutable
public final class SourceManifestAction extends AbstractFileWriteAction {

  private static final String GUID = "07459553-a3d0-4d37-9d78-18ed942470f4";

  /**
   * Interface for defining manifest formatting and reporting specifics. Implementations must be
   * immutable.
   */
  @VisibleForTesting
  interface ManifestWriter {

    /**
     * Writes a single line of manifest output.
     *
     * @param manifestWriter the output stream
     * @param rootRelativePath path of an entry relative to the manifest's root
     * @param symlink (optional) symlink that resolves the above path
     */
    void writeEntry(Writer manifestWriter, PathFragment rootRelativePath,
        @Nullable Artifact symlink) throws IOException;

    /**
     * Fulfills {@link com.google.devtools.build.lib.actions.AbstractAction#getMnemonic()}
     */
    String getMnemonic();

    /**
     * Fulfills {@link com.google.devtools.build.lib.actions.AbstractAction#getRawProgressMessage()}
     */
    String getRawProgressMessage();

    /**
     * Fulfills {@link AbstractFileWriteAction#isRemotable()}.
     * @return
     */
    boolean isRemotable();
  }

  /**
   * The strategy we use to write manifest entries.
   */
  private final ManifestWriter manifestWriter;

  /**
   * The runfiles for which to create the symlink tree.
   */
  private final Runfiles runfiles;

  /**
   * Creates a new AbstractSourceManifestAction instance using latin1 encoding to write the manifest
   * file and with a specified root path for manifest entries.
   *
   * @param manifestWriter the strategy to use to write manifest entries
   * @param owner the action owner
   * @param primaryOutput the file to which to write the manifest
   * @param runfiles runfiles
   */
  @VisibleForSerialization
  SourceManifestAction(
      ManifestWriter manifestWriter, ActionOwner owner, Artifact primaryOutput, Runfiles runfiles) {
    super(owner, getDependencies(runfiles), primaryOutput, false);
    this.manifestWriter = manifestWriter;
    this.runfiles = runfiles;
  }

  /** The {@link Runfiles} for which this action creates the symlink tree. */
  public Runfiles getGeneratedRunfiles() {
    return runfiles;
  }

  @VisibleForTesting
  public void writeOutputFile(OutputStream out, EventHandler eventHandler)
      throws IOException {
    writeFile(out,
        runfiles.getRunfilesInputs(
            eventHandler,
            getOwner().getLocation(),
            ArtifactPathResolver.IDENTITY));
  }

  @Override
  public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx)
      throws IOException {
    final Map<PathFragment, Artifact> runfilesInputs =
        runfiles.getRunfilesInputs(ctx.getEventHandler(), getOwner().getLocation(),
            ctx.getPathResolver());
    return out -> writeFile(out, runfilesInputs);
  }

  @Override
  public boolean isRemotable() {
    return manifestWriter.isRemotable();
  }

  /**
   * Returns the input dependencies for this action. Note we don't need to create the symlink
   * target Artifacts before we write the output manifest, so this Action does not have to
   * depend on them. The only necessary dependencies are pruning manifests, which must be read
   * to properly prune the tree.
   */
  public static Collection<Artifact> getDependencies(Runfiles runfiles) {
    ImmutableList.Builder<Artifact> builder = ImmutableList.builder();
    for (Runfiles.PruningManifest manifest : runfiles.getPruningManifests()) {
      builder.add(manifest.getManifestFile());
    }
    return builder.build();
  }

  /**
   * Sort the entries in both the normal and root manifests and write the output
   * file.
   *
   * @param out is the message stream to write errors to.
   * @param output The actual mapping of the output manifest.
   * @throws IOException
   */
  private void writeFile(OutputStream out, Map<PathFragment, Artifact> output) throws IOException {
    Writer manifestFile = new BufferedWriter(new OutputStreamWriter(out, ISO_8859_1));

    Comparator<Map.Entry<PathFragment, Artifact>> fragmentComparator =
          new Comparator<Map.Entry<PathFragment, Artifact>>() {
      @Override
      public int compare(Map.Entry<PathFragment, Artifact> path1,
                         Map.Entry<PathFragment, Artifact> path2) {
        return path1.getKey().compareTo(path2.getKey());
      }
    };

    List<Map.Entry<PathFragment, Artifact>> sortedManifest = new ArrayList<>(output.entrySet());
    Collections.sort(sortedManifest, fragmentComparator);

    for (Map.Entry<PathFragment, Artifact> line : sortedManifest) {
      manifestWriter.writeEntry(manifestFile, line.getKey(), line.getValue());
    }

    manifestFile.flush();
  }

  @Override
  public String getMnemonic() {
    return manifestWriter.getMnemonic();
  }

  @Override
  protected String getRawProgressMessage() {
    return manifestWriter.getRawProgressMessage() + " for " + getOwner().getLabel();
  }

  @Override
  protected void computeKey(ActionKeyContext actionKeyContext, Fingerprint fp) {
    fp.addString(GUID);
    runfiles.fingerprint(fp);
  }

  /**
   * Supported manifest writing strategies.
   */
  public enum ManifestType implements ManifestWriter {

    /**
     * Writes each line as:
     *
     * [rootRelativePath] [resolvingSymlink]
     *
     * <p>This strategy is suitable for creating an input manifest to a source view tree. Its
     * output is a valid input to
     * {@link com.google.devtools.build.lib.analysis.actions.SymlinkTreeAction}.
     */
    SOURCE_SYMLINKS {
      @Override
      public void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, Artifact symlink)
          throws IOException {
        manifestWriter.append(rootRelativePath.getPathString());
        // This trailing whitespace is REQUIRED to process the single entry line correctly.
        manifestWriter.append(' ');
        if (symlink != null) {
          manifestWriter.append(symlink.getPath().getPathString());
        }
        manifestWriter.append('\n');
      }

      @Override
      public String getMnemonic() {
        return "SourceSymlinkManifest";
      }

      @Override
      public String getRawProgressMessage() {
        return "Creating source manifest";
      }

      @Override
      public boolean isRemotable() {
        // There is little gain to remoting these, since they include absolute path names inline.
        return false;
      }
    },

    /**
     * Writes each line as:
     *
     * [rootRelativePath]
     *
     * <p>This strategy is suitable for an input into a packaging system (notably .par) that
     * consumes a list of all source files but needs that list to be constant with respect to
     * how the user has their client laid out on local disk.
     */
    SOURCES_ONLY {
      @Override
      public void writeEntry(Writer manifestWriter, PathFragment rootRelativePath, Artifact symlink)
          throws IOException {
        manifestWriter.append(rootRelativePath.getPathString());
        manifestWriter.append('\n');
        manifestWriter.flush();
      }

      @Override
      public String getMnemonic() {
        return "PackagingSourcesManifest";
      }

      @Override
      public String getRawProgressMessage() {
        return "Creating file sources list";
      }

      @Override
      public boolean isRemotable() {
        // Source-only symlink manifest has root-relative paths and does not include absolute paths.
        return true;
      }
    }
  }

  /** Creates an action for the given runfiles. */
  public static SourceManifestAction forRunfiles(ManifestType manifestType, ActionOwner owner,
      Artifact output, Runfiles runfiles) {
    return new SourceManifestAction(manifestType, owner, output, runfiles);
  }

  /**
   * Builder class to construct {@link SourceManifestAction} instances.
   */
  public static final class Builder {
    private final ManifestWriter manifestWriter;
    private final ActionOwner owner;
    private final Artifact output;
    private final Runfiles.Builder runfilesBuilder;

    public Builder(String prefix, ManifestType manifestType, ActionOwner owner, Artifact output,
                   boolean legacyExternalRunfiles) {
      this(manifestType, owner, output, new Runfiles.Builder(prefix, legacyExternalRunfiles));
    }

    public Builder(
        ManifestType manifestType,
        ActionOwner owner,
        Artifact output,
        Runfiles.Builder runfilesBuilder) {
      this.manifestWriter = manifestType;
      this.owner = owner;
      this.output = output;
      this.runfilesBuilder = runfilesBuilder;
    }

    @VisibleForTesting  // Only used for testing.
    Builder(String prefix, ManifestWriter manifestWriter, ActionOwner owner, Artifact output) {
      this.runfilesBuilder = new Runfiles.Builder(prefix, false);
      this.manifestWriter = manifestWriter;
      this.owner = owner;
      this.output = output;
    }

    public SourceManifestAction build() {
      return new SourceManifestAction(manifestWriter, owner, output, runfilesBuilder.build());
    }

    /**
     * Adds a set of symlinks from the artifacts' root-relative paths to the
     * artifacts themselves.
     */
    public Builder addSymlinks(Iterable<Artifact> artifacts) {
      runfilesBuilder.addArtifacts(artifacts);
      return this;
    }

    /**
     * Adds a map of symlinks.
     */
    public Builder addSymlinks(Map<PathFragment, Artifact> symlinks) {
      runfilesBuilder.addSymlinks(symlinks);
      return this;
    }

    /**
     * Adds a single symlink.
     */
    public Builder addSymlink(PathFragment link, Artifact target) {
      runfilesBuilder.addSymlink(link, target);
      return this;
    }

    /**
     * <p>Adds a mapping of Artifacts to the directory above the normal symlink
     * forest base.
     */
    public Builder addRootSymlinks(Map<PathFragment, Artifact> rootSymlinks) {
      runfilesBuilder.addRootSymlinks(rootSymlinks);
      return this;
    }

    /**
     * Set the empty files supplier for the manifest, see {@link Runfiles.EmptyFilesSupplier}
     * for more details.
     */
    public Builder setEmptyFilesSupplier(Runfiles.EmptyFilesSupplier supplier) {
      runfilesBuilder.setEmptyFilesSupplier(supplier);
      return this;
    }

    /**
     * Adds a runfiles pruning manifest.
     */
    @VisibleForTesting
    Builder addPruningManifest(Runfiles.PruningManifest manifest) {
      runfilesBuilder.addPruningManifest(manifest);
      return this;
    }
  }
}