src/main/java/com/google/devtools/build/lib/skyframe/ActionInputMapHelper.java

// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
import com.google.devtools.build.lib.actions.ActionInputMapSink;
import com.google.devtools.build.lib.actions.ActionLookupData;
import com.google.devtools.build.lib.actions.ActionLookupKey;
import com.google.devtools.build.lib.actions.ActionLookupValue;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.ArchivedTreeArtifact;
import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
import com.google.devtools.build.lib.analysis.actions.SymlinkAction;
import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyValue;
import java.util.Map;
import javax.annotation.Nullable;

/** Static utilities for working with action inputs. */
final class ActionInputMapHelper {

  private ActionInputMapHelper() {}

  static void addToMap(
      ActionInputMapSink inputMap,
      Map<Artifact, ImmutableCollection<? extends Artifact>> expandedArtifacts,
      Map<SpecialArtifact, ArchivedTreeArtifact> archivedTreeArtifacts,
      Map<Artifact, ImmutableList<FilesetOutputSymlink>> filesetsInsideRunfiles,
      Map<Artifact, ImmutableList<FilesetOutputSymlink>> topLevelFilesets,
      Artifact key,
      SkyValue value,
      Environment env,
      boolean prefetcherSupportsPartialTreeArtifacts)
      throws InterruptedException {
    addToMap(
        inputMap,
        expandedArtifacts,
        archivedTreeArtifacts,
        filesetsInsideRunfiles,
        topLevelFilesets,
        key,
        value,
        env,
        MetadataConsumerForMetrics.NO_OP,
        prefetcherSupportsPartialTreeArtifacts);
  }

  /**
   * Adds a value obtained by an Artifact skyvalue lookup to the action input map. May do Skyframe
   * lookups.
   */
  static void addToMap(
      ActionInputMapSink inputMap,
      Map<Artifact, ImmutableCollection<? extends Artifact>> expandedArtifacts,
      Map<SpecialArtifact, ArchivedTreeArtifact> archivedTreeArtifacts,
      Map<Artifact, ImmutableList<FilesetOutputSymlink>> filesetsInsideRunfiles,
      Map<Artifact, ImmutableList<FilesetOutputSymlink>> topLevelFilesets,
      Artifact key,
      SkyValue value,
      Environment env,
      MetadataConsumerForMetrics consumer,
      boolean prefetcherSupportsPartialTreeArtifacts)
      throws InterruptedException {
    if (value instanceof RunfilesArtifactValue) {
      // Note: we don't expand the .runfiles/MANIFEST file into the inputs. The reason for that
      // being that the MANIFEST file contains absolute paths that don't work with remote execution.
      // Instead, the way the SpawnInputExpander expands runfiles is via the Runfiles class
      // which contains all artifacts in the runfiles tree minus the MANIFEST file.
      RunfilesArtifactValue runfilesArtifactValue = (RunfilesArtifactValue) value;
      for (Pair<Artifact, FileArtifactValue> entry : runfilesArtifactValue.getFileArtifacts()) {
        Artifact artifact = entry.first;
        inputMap.put(artifact, entry.getSecond(), /*depOwner=*/ key);
        if (artifact.isFileset()) {
          ImmutableList<FilesetOutputSymlink> expandedFileset =
              getFilesets(env, (SpecialArtifact) artifact);
          if (expandedFileset != null) {
            filesetsInsideRunfiles.put(artifact, expandedFileset);
            consumer.accumulate(expandedFileset);
          }
        } else {
          consumer.accumulate(entry.getSecond());
        }
      }
      for (Pair<Artifact, TreeArtifactValue> entry : runfilesArtifactValue.getTreeArtifacts()) {
        expandTreeArtifactAndPopulateArtifactData(
            entry.getFirst(),
            Preconditions.checkNotNull(entry.getSecond()),
            expandedArtifacts,
            archivedTreeArtifacts,
            inputMap,
            /*depOwner=*/ key);
        consumer.accumulate(entry.getSecond());
      }
      // We have to cache the "digest" of the aggregating value itself, because the action cache
      // checker may want it.
      inputMap.put(key, runfilesArtifactValue.getMetadata(), /*depOwner=*/ key);
    } else if (value instanceof TreeArtifactValue) {
      TreeArtifactValue treeArtifactValue = (TreeArtifactValue) value;
      expandTreeArtifactAndPopulateArtifactData(
          key,
          treeArtifactValue,
          expandedArtifacts,
          archivedTreeArtifacts,
          inputMap,
          /*depOwner=*/ key);
      consumer.accumulate(treeArtifactValue);
    } else if (value instanceof ActionExecutionValue) {
      if (!prefetcherSupportsPartialTreeArtifacts && key instanceof TreeFileArtifact) {
        // If we're unable to prefetch individual files in a tree artifact, include the full tree
        // artifact in the action inputs. This makes actions that consume partial tree artifacts
        // (such as the ones generated by SpawnActionTemplate or CppCompileActionTemplate) less
        // efficient, but is needed until https://github.com/bazelbuild/bazel/issues/16333 is fixed.
        SpecialArtifact treeArtifact = key.getParent();
        TreeArtifactValue treeArtifactValue =
            ((ActionExecutionValue) value).getTreeArtifactValue(treeArtifact);
        expandTreeArtifactAndPopulateArtifactData(
            treeArtifact,
            treeArtifactValue,
            expandedArtifacts,
            archivedTreeArtifacts,
            inputMap,
            /* depOwner= */ treeArtifact);
        consumer.accumulate(treeArtifactValue);
      } else {
        FileArtifactValue metadata =
            ((ActionExecutionValue) value).getExistingFileArtifactValue(key);
        inputMap.put(key, metadata, key);
        if (key.isFileset()) {
          ImmutableList<FilesetOutputSymlink> filesets = getFilesets(env, (SpecialArtifact) key);
          if (filesets != null) {
            topLevelFilesets.put(key, filesets);
            consumer.accumulate(filesets);
          }
        } else {
          consumer.accumulate(metadata);
        }
      }
    } else {
      Preconditions.checkArgument(value instanceof FileArtifactValue, "Unexpected value %s", value);
      FileArtifactValue metadata = (FileArtifactValue) value;
      inputMap.put(key, metadata, /*depOwner=*/ key);
      consumer.accumulate(metadata);
    }
  }

  @Nullable
  static ImmutableList<FilesetOutputSymlink> getFilesets(
      Environment env, SpecialArtifact actionInput) throws InterruptedException {
    Preconditions.checkState(actionInput.isFileset(), actionInput);
    ActionLookupData generatingActionKey = actionInput.getGeneratingActionKey();
    ActionLookupKey filesetActionLookupKey = generatingActionKey.getActionLookupKey();

    ActionLookupValue filesetActionLookupValue =
        (ActionLookupValue) env.getValue(filesetActionLookupKey);

    ActionAnalysisMetadata generatingAction =
        filesetActionLookupValue.getAction(generatingActionKey.getActionIndex());
    ActionLookupData filesetActionKey;

    if (generatingAction instanceof SymlinkAction) {
      DerivedArtifact outputManifest =
          (DerivedArtifact) generatingAction.getInputs().getSingleton();
      ActionLookupData manifestGeneratingKey = outputManifest.getGeneratingActionKey();
      Preconditions.checkState(
          manifestGeneratingKey.getActionLookupKey().equals(filesetActionLookupKey),
          "Mismatched actions and artifacts: %s %s %s %s",
          actionInput,
          outputManifest,
          filesetActionLookupKey,
          manifestGeneratingKey);
      ActionAnalysisMetadata symlinkTreeAction =
          filesetActionLookupValue.getAction(manifestGeneratingKey.getActionIndex());
      DerivedArtifact inputManifest =
          (DerivedArtifact) symlinkTreeAction.getInputs().getSingleton();
      ActionLookupData inputManifestGeneratingKey = inputManifest.getGeneratingActionKey();
      Preconditions.checkState(
          inputManifestGeneratingKey.getActionLookupKey().equals(filesetActionLookupKey),
          "Mismatched actions and artifacts: %s %s %s %s",
          actionInput,
          inputManifest,
          filesetActionLookupKey,
          inputManifestGeneratingKey);
      filesetActionKey = inputManifestGeneratingKey;
    } else {
      filesetActionKey = generatingActionKey;
    }

    // TODO(janakr: properly handle exceptions coming from here, or prove they can never happen in
    //  practice.
    ActionExecutionValue filesetValue = (ActionExecutionValue) env.getValue(filesetActionKey);
    if (filesetValue == null) {
      // At this point skyframe does not guarantee that the filesetValue will be ready, since
      // the current action does not directly depend on the outputs of the
      // SkyframeFilesetManifestAction whose ActionExecutionValue (filesetValue) is needed here.
      return null;
    }
    return filesetValue.getOutputSymlinks();
  }

  private static void expandTreeArtifactAndPopulateArtifactData(
      Artifact treeArtifact,
      TreeArtifactValue value,
      Map<Artifact, ImmutableCollection<? extends Artifact>> expandedArtifacts,
      Map<SpecialArtifact, ArchivedTreeArtifact> archivedTreeArtifacts,
      ActionInputMapSink inputMap,
      Artifact depOwner) {
    if (TreeArtifactValue.OMITTED_TREE_MARKER.equals(value)) {
      inputMap.putTreeArtifact((SpecialArtifact) treeArtifact, value, depOwner);
      return;
    }

    inputMap.putTreeArtifact((SpecialArtifact) treeArtifact, value, depOwner);
    expandedArtifacts.put(treeArtifact, value.getChildren());

    value
        .getArchivedRepresentation()
        .ifPresent(
            archivedRepresentation -> {
              inputMap.put(
                  archivedRepresentation.archivedTreeFileArtifact(),
                  archivedRepresentation.archivedFileValue(),
                  depOwner);
              archivedTreeArtifacts.put(
                  (SpecialArtifact) treeArtifact,
                  archivedRepresentation.archivedTreeFileArtifact());
            });
  }
}