src/main/java/com/google/devtools/build/lib/analysis/actions/PopulateTreeArtifactAction.java

// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.analysis.actions;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.AbstractAction;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionExecutionException;
import com.google.devtools.build.lib.actions.ActionExecutionMetadata;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionInputHelper;
import com.google.devtools.build.lib.actions.ActionOwner;
import com.google.devtools.build.lib.actions.Actions;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException;
import com.google.devtools.build.lib.actions.BaseSpawn;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.ResourceSet;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnActionContext;
import com.google.devtools.build.lib.analysis.FilesToRunProvider;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.PathFragment;

import java.io.IOException;
import java.util.Collection;
import java.util.Map;

/**
 * An action that populates a TreeArtifact with the contents of an archive file.
 *
 * <p>Internally, the following happens at execution time:
 * <ol>
 *   <li>The archive entry paths are read from the associated archive manifest file locally.
 *   <li>A spawn is executed to unzip the archive contents under the root of the TreeArtifact.
 *   <li>Child TreeFileArtifacts are created using the archive entry paths and then populated into
 *       the output TreeArtifact.
 * </ol>
 *
 * <p>There are also several requirements regarding the archive and archive manifest file:
 * <ul>
 *   <li>The entry names (paths) of the archive and archive manifest file must be valid ISO-8859-1
 *       strings.
 *   <li>The archive manifest file must not contain absolute, non-normalized
 *       (e.g., containing '..' fragments) or duplicated paths. And no path is allowed to be a
 *       prefix of another path.
 * </ul>
 */
public final class PopulateTreeArtifactAction extends AbstractAction {
  private static final String GUID = "a3d36f29-9f14-42cf-a014-3a51e914e482";

  @VisibleForTesting
  static final String MNEMONIC = "PopulateTreeArtifact";

  private final Artifact archive;
  private final Artifact archiveManifest;
  private final Artifact outputTreeArtifact;
  private final FilesToRunProvider zipper;

  /**
   * Creates a PopulateTreeArtifactAction object.
   *
   * @param owner the owner of the action.
   * @param archive the archive containing files to populate into the TreeArtifact.
   * @param archiveManifest the archive manifest file specifying the entry files to populate into
   *     the TreeArtifact.
   * @param treeArtifactToPopulate the TreeArtifact to be populated with archive member files.
   * @param zipper the zipper executable used to unzip the archive.
   */
  public PopulateTreeArtifactAction(
      ActionOwner owner,
      Artifact archive,
      Artifact archiveManifest,
      Artifact treeArtifactToPopulate,
      FilesToRunProvider zipper) {
    super(
        owner,
        ImmutableList.copyOf(zipper.getFilesToRun()),
        Iterables.concat(
            ImmutableList.of(archive, archiveManifest),
            ImmutableList.copyOf(zipper.getFilesToRun())),
        ImmutableList.of(treeArtifactToPopulate));

    Preconditions.checkArgument(
        treeArtifactToPopulate.isTreeArtifact(),
        "%s is not TreeArtifact",
        treeArtifactToPopulate);

    this.archive = archive;
    this.archiveManifest = archiveManifest;
    this.outputTreeArtifact = treeArtifactToPopulate;
    this.zipper = zipper;
  }

  private static class PopulateTreeArtifactSpawn extends BaseSpawn {
    private final Artifact treeArtifact;
    private final Iterable<PathFragment> entriesToExtract;

    // The output TreeFileArtifacts are created lazily outside of the contructor because potentially
    // we can have a lot of TreeFileArtifacts under a given tree artifact.
    private Collection<TreeFileArtifact> outputTreeFileArtifacts;

    PopulateTreeArtifactSpawn(
        Artifact treeArtifact,
        Iterable<PathFragment> entriesToExtract,
        Iterable<String> commandLine,
        Map<PathFragment, Artifact> runfilesManifests,
        ActionExecutionMetadata action) {
      super(
        ImmutableList.copyOf(commandLine),
        ImmutableMap.<String, String>of(),
        ImmutableMap.<String, String>of(),
        ImmutableMap.copyOf(runfilesManifests),
        action,
        AbstractAction.DEFAULT_RESOURCE_SET);
      this.treeArtifact = treeArtifact;
      this.entriesToExtract = entriesToExtract;
    }

    @Override
    public Collection<? extends ActionInput> getOutputFiles() {
      if (outputTreeFileArtifacts == null) {
        outputTreeFileArtifacts = ImmutableList.<TreeFileArtifact>copyOf(
            ActionInputHelper.asTreeFileArtifacts(treeArtifact, entriesToExtract));
      }
      return outputTreeFileArtifacts;
    }
  }

  @Override
  public Artifact getPrimaryOutput() {
    return outputTreeArtifact;
  }

  @Override
  public void execute(ActionExecutionContext actionExecutionContext)
      throws ActionExecutionException, InterruptedException {
    Executor executor = actionExecutionContext.getExecutor();
    Spawn spawn;

    // Create a spawn to unzip the archive file into the output TreeArtifact.
    try {
      spawn = createSpawn();
    } catch (IOException e) {
      throw new ActionExecutionException(e, this, false);
    } catch (IllegalManifestFileException e) {
      throw new ActionExecutionException(e, this, true);
    }

    // If the spawn does not have any output, it means the archive file contains nothing. In this
    // case we just return without generating anything under the output TreeArtifact.
    if (spawn.getOutputFiles().isEmpty()) {
      return;
    }

    // Check spawn output TreeFileArtifact conflicts.
    try {
      checkOutputConflicts(spawn.getOutputFiles());
    } catch (ArtifactPrefixConflictException e) {
      throw new ActionExecutionException(e, this, true);
    }

    // Create parent directories for the output TreeFileArtifacts.
    try {
      for (ActionInput fileEntry : spawn.getOutputFiles()) {
        FileSystemUtils.createDirectoryAndParents(
            ((Artifact) fileEntry).getPath().getParentDirectory());
      }
    } catch (IOException e) {
      throw new ActionExecutionException(e, this, false);
    }

    // Execute the spawn.
    try {
      getContext(executor).exec(spawn, actionExecutionContext);
    } catch (ExecException e) {
      throw e.toActionExecutionException(
          getMnemonic() + " action failed for target: " + getOwner().getLabel(),
          executor.getVerboseFailures(),
          this);
    }

    // Populate the output TreeArtifact with the Spawn output TreeFileArtifacts.
    for (ActionInput fileEntry : spawn.getOutputFiles()) {
      actionExecutionContext.getMetadataHandler().addExpandedTreeOutput(
          (TreeFileArtifact) fileEntry);
    }
  }

  @Override
  protected String computeKey() {
    Fingerprint f = new Fingerprint();
    f.addString(GUID);
    f.addString(getMnemonic());
    f.addStrings(spawnCommandLine());
    Map<PathFragment, Artifact> zipperManifest = zipperExecutableRunfilesManifest();
    f.addInt(zipperManifest.size());
    for (Map.Entry<PathFragment, Artifact> input : zipperManifest.entrySet()) {
      f.addString(input.getKey().getPathString() + "/");
      f.addPath(input.getValue().getExecPath());
    }

    return f.hexDigestAndReset();
  }

  @Override
  public String getMnemonic() {
    return "PopulateTreeArtifact";
  }

  @Override
  public boolean shouldReportPathPrefixConflict(ActionAnalysisMetadata action) {
    return true;
  }

  @Override
  public ResourceSet estimateResourceConsumption(Executor executor) {
    if (getContext(executor).willExecuteRemotely(true)) {
      return ResourceSet.ZERO;
    }
    return AbstractAction.DEFAULT_RESOURCE_SET;
  }

  private SpawnActionContext getContext(Executor executor) {
    return executor.getSpawnActionContext(getMnemonic());
  }

  /**
   * Creates a spawn to unzip the archive members specified in the archive manifest into the
   * TreeArtifact.
   */
  @VisibleForTesting
  Spawn createSpawn() throws IOException, IllegalManifestFileException {
    Iterable<PathFragment> entries = readAndCheckManifestEntries();
    return new PopulateTreeArtifactSpawn(
        outputTreeArtifact,
        entries,
        spawnCommandLine(),
        zipperExecutableRunfilesManifest(),
        this);
  }

  private Iterable<String> spawnCommandLine() {
    return ImmutableList.of(
        zipper.getExecutable().getExecPathString(),
        "x",
        archive.getExecPathString(),
        "-d",
        outputTreeArtifact.getExecPathString(),
        "@" + archiveManifest.getExecPathString());
  }

  private Map<PathFragment, Artifact> zipperExecutableRunfilesManifest() {
    if (zipper.getRunfilesManifest() != null) {
      return ImmutableMap.of(
          BaseSpawn.runfilesForFragment(zipper.getExecutable().getExecPath()),
          zipper.getRunfilesManifest());
    } else {
      return ImmutableMap.<PathFragment, Artifact>of();
    }
  }

  private Iterable<PathFragment> readAndCheckManifestEntries()
      throws IOException, IllegalManifestFileException {
    ImmutableList.Builder<PathFragment> manifestEntries = ImmutableList.builder();

    for (String line :
        FileSystemUtils.iterateLinesAsLatin1(archiveManifest.getPath())) {
      if (!line.isEmpty()) {
        PathFragment path = new PathFragment(line);

        if (!path.isNormalized() || path.isAbsolute()) {
          throw new IllegalManifestFileException(
              path + " is not a proper relative path");
        }

        manifestEntries.add(path);
      }
    }

    return manifestEntries.build();
  }

  private void checkOutputConflicts(Collection<? extends ActionInput> outputs)
      throws ArtifactPrefixConflictException {
    ImmutableMap.Builder<Artifact, ActionAnalysisMetadata> generatingActions =
        ImmutableMap.<Artifact, ActionAnalysisMetadata>builder();
    for (ActionInput output : outputs) {
      generatingActions.put((Artifact) output, this);
    }

    Map<ActionAnalysisMetadata, ArtifactPrefixConflictException> artifactPrefixConflictMap =
        Actions.findArtifactPrefixConflicts(generatingActions.build());

    if (!artifactPrefixConflictMap.isEmpty()) {
      throw artifactPrefixConflictMap.values().iterator().next();
    }
  }

  private static class IllegalManifestFileException extends Exception {

    IllegalManifestFileException(String message) {
      super(message);
    }
  }
}