src/main/java/com/google/devtools/build/lib/actions/PathStripper.java - bazel - Git at Google

 // Copyright 2021 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package com.google.devtools.build.lib.actions;

 import static com.google.common.collect.ImmutableList.toImmutableList;

 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.actions.CommandLines.ParamFileActionInput;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.regex.Pattern;

 /**
  * Main logic for experimental config-stripped execution paths:
  * https://github.com/bazelbuild/bazel/issues/6526.
  *
  * <p>The actions executors run look like: {@code tool_pkg/mytool src/source.file
  * bazel-out/x86-opt/pkg/gen.file -o bazel-out/x86-opt/pkg/myout}.
  *
  * <p>The "x86-opt" part is a path's "configuration prefix": information describing the build
  * configuration of the action creating the artifact. This example shows artifacts created with
  * {@code --cpu=x86 --compilation_mode=opt}.
  *
  * <p>Executors cache actions based on their a) command line, b) input and output paths, c) input
  * digests. Configuration prefixes harm caching because even if an action behaves exactly the same
  * for different CPU architectures, {@code <cpu>-opt} guarantees the paths will differ.
  *
  * <p>Config-stripping is an experimental feature that strips the configuration prefix from
  * qualifying actions before running them, thus improving caching. "Qualifying" actions are actions
  * known not to depend on the names of their input and output paths. Non-qualifying actions include
  * manifest generators and compilers that store debug symbol source paths.
  *
  * <p>As an experimental feature, most logic is centralized here to provide easy hooks into executor
  * code and avoid complicating large swaths of the code base. "Qualifying" actions are determined by
  * {@code --experimental_path_agnostic_action}. As this feature stabilizes, rule definitions can
  * more directly own their integration, particularly declaring which of their actions qualify and
  * where paths appear in their command lines.
  */
 public interface PathStripper {
   /**
    * Returns the exec path a running action should use to identify one of its inputs or outputs.
    *
    * <p>If the action should be config-stripped ({@link PathStripper}), removes "k8-fastbuild" from
    * paths like "bazel-out/k8-fastbuild/foo/bar".
    *
    * <p>Else returns the artifact's original exec path.
    */
   String getExecPathString(ActionInput artifact);

   /**
    * If this action strips paths, textually replaces "bazel-out/x86-fastbuild/foo/..." paths in a
    * command line argument with "bazel-out/foo/...". Else returns the input as-is.
    *
    * @param arg a string expected to be part of the action's command line.
    */
   String processCmdArg(String arg);

   /**
    * Adjusts a set of action inputs for possible config path stripping.
    *
    * <p>For artifacts, this is a no-op.
    *
    * <p>{@link com.google.devtools.build.lib.actions.cache.VirtualActionInput}s materialize on
    * demand, so their behavior may be different. {@link ParamFileActionInput}, particularly, stores
    * a command line in a file. So any paths in that command line changes that input's contents.
    * That's the use case this method handles.
    */
   List<ActionInput> processInputs(List<ActionInput> inputs);

   /** Instantiates a {@link PathStripper} that doesn't change paths. */
   static PathStripper noop() {
     return new PathStripper() {
       @Override
       public String getExecPathString(ActionInput artifact) {
         return artifact.getExecPathString();
       }

       @Override
       public String processCmdArg(String arg) {
         return arg;
       }

       @Override
       public List<ActionInput> processInputs(List<ActionInput> inputs) {
         return inputs;
       }
     };
   }

   /**
    * Instantiates a {@link PathStripper} for a spawn action.
    *
    * @param spawn the action to support
    * @param pathAgnosticActions mnemonics of actions that "qualify" for path stripping. Just because
    *     an action type qualifies doesn't mean its paths are stripped. See {@link
    *     #shouldStripPaths}.
    * @param outputRoot root of the output tree ("bazel-out").
    */
   static PathStripper create(
       Spawn spawn, Collection<String> pathAgnosticActions, PathFragment outputRoot) {
     Preconditions.checkState(outputRoot.isSingleSegment());
     Preconditions.checkState(!outputRoot.getPathString().contains("\\"));
     if (!shouldStripPaths(spawn, pathAgnosticActions, outputRoot)) {
       return noop();
     }
     Pattern stripPathsPattern =
         Pattern.compile("\\Q" + outputRoot.getPathString() + "\\E/[\\w-_]+(/|$)");

     return new PathStripper() {
       @Override
       public String getExecPathString(ActionInput artifact) {
         if (!isOutputPath(artifact, outputRoot)) {
           return artifact.getExecPathString();
         }
         PathFragment origExecPath = artifact.getExecPath();
         return outputRoot.getRelative(origExecPath.subFragment(2)).toString();
       }

       @Override
       public String processCmdArg(String arg) {
         // Note that we can't just split the input on a simple delimiter like " ". Output paths
         // can be prefixed by all kinds of characters. Examples:
         //   params files: "@bazel-out/..."
         //   AndroidResourceCompiler: "--resource some_path#bazel-out/x86-opt/some-other-path"
         //   busybox.bzl (lots of Android actions): paths prefixed with ",", ":", and more
         //
         // A deeper and probably more sustainable design would be for all actions to store path
         // references as structured inputs (e.g. an Artifact object instead of a path string).
         // Then lazily instantiate their paths in the executor client. That's roughly what
         // processInputs(), below, models for ParamFileActionInputs. But that involves more API
         // modeling and rule logic cleanup. So we take the path of least resistance for now.
         return stripPathsPattern.matcher(arg).replaceAll(outputRoot.getPathString() + "/");
       }

       @Override
       public ImmutableList<ActionInput> processInputs(List<ActionInput> inputs) {
         return inputs.stream()
             .map(
                 input ->
                     input instanceof ParamFileActionInput
                         ? ((ParamFileActionInput) input).withAdjustedArgs(this::processCmdArg)
                         : input)
             .collect(toImmutableList());
       }
     };
   }

   /**
    * Should this action have its paths stripped for execution?
    *
    * <p>Only true for actions that a) qualify, b) don't have distinct paths that would become
    * duplicates if their config prefixes were removed.
    */
   static boolean shouldStripPaths(
       Spawn spawn, Collection<String> pathAgnosticActions, PathFragment outputRoot) {
     String actionMnemonic = spawn.getMnemonic();
     if (!pathAgnosticActions.contains(actionMnemonic)) {
       return false;
     }

     // For qualifying action types, check that no inputs or outputs would clash if paths were
     // removed, e.g. "bazel-out/k8-fastbuild/foo" and "bazel-out/host/foo".
     //
     // A more clever algorithm could remap these with custom prefixes - "bazel-out/1/foo" and
     // "bazel-out/2/foo" - if experience shows that would help.
     //
     // Another approach could keep host paths intact (since the "host" path prefix doesn't vary
     // with configurations). While this would help more action instances qualify, it also blocks
     // caching the same action in host and target configurations. This could be mitigated by
     // stripping the host prefix *only* when the entire action is in the host configuration.
     HashSet<PathFragment> rootRelativePaths = new HashSet<>();
     for (ActionInput input : spawn.getInputFiles().toList()) {
       if (!isOutputPath(input, outputRoot)) {
         continue;
       }
       // For "bazel-out/k8-fastbuild/foo/bar", get "foo/bar".
       if (!rootRelativePaths.add(input.getExecPath().subFragment(2))) {
         return false;
       }
     }
     return true;
   }

   /** Is this a strippable path? */
   static boolean isOutputPath(ActionInput artifact, PathFragment outputRoot) {
     // We can't simply check for DerivedArtifact. Output paths can also appear, for example, in
     // ParamFileActionInput and ActionInputHelper.BasicActionInput.
     return artifact.getExecPath().startsWith(outputRoot);
   }
 }
	// Copyright 2021 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package com.google.devtools.build.lib.actions;

	import static com.google.common.collect.ImmutableList.toImmutableList;

	import com.google.common.base.Preconditions;
	import com.google.common.collect.ImmutableList;
	import com.google.devtools.build.lib.actions.CommandLines.ParamFileActionInput;
	import com.google.devtools.build.lib.vfs.PathFragment;
	import java.util.Collection;
	import java.util.HashSet;
	import java.util.List;
	import java.util.regex.Pattern;

	/**
	* Main logic for experimental config-stripped execution paths:
	* https://github.com/bazelbuild/bazel/issues/6526.
	*
	* <p>The actions executors run look like: {@code tool_pkg/mytool src/source.file
	* bazel-out/x86-opt/pkg/gen.file -o bazel-out/x86-opt/pkg/myout}.
	*
	* <p>The "x86-opt" part is a path's "configuration prefix": information describing the build
	* configuration of the action creating the artifact. This example shows artifacts created with
	* {@code --cpu=x86 --compilation_mode=opt}.
	*
	* <p>Executors cache actions based on their a) command line, b) input and output paths, c) input
	* digests. Configuration prefixes harm caching because even if an action behaves exactly the same
	* for different CPU architectures, {@code <cpu>-opt} guarantees the paths will differ.
	*
	* <p>Config-stripping is an experimental feature that strips the configuration prefix from
	* qualifying actions before running them, thus improving caching. "Qualifying" actions are actions
	* known not to depend on the names of their input and output paths. Non-qualifying actions include
	* manifest generators and compilers that store debug symbol source paths.
	*
	* <p>As an experimental feature, most logic is centralized here to provide easy hooks into executor
	* code and avoid complicating large swaths of the code base. "Qualifying" actions are determined by
	* {@code --experimental_path_agnostic_action}. As this feature stabilizes, rule definitions can
	* more directly own their integration, particularly declaring which of their actions qualify and
	* where paths appear in their command lines.
	*/
	public interface PathStripper {
	/**
	* Returns the exec path a running action should use to identify one of its inputs or outputs.
	*
	* <p>If the action should be config-stripped ({@link PathStripper}), removes "k8-fastbuild" from
	* paths like "bazel-out/k8-fastbuild/foo/bar".
	*
	* <p>Else returns the artifact's original exec path.
	*/
	String getExecPathString(ActionInput artifact);

	/**
	* If this action strips paths, textually replaces "bazel-out/x86-fastbuild/foo/..." paths in a
	* command line argument with "bazel-out/foo/...". Else returns the input as-is.
	*
	* @param arg a string expected to be part of the action's command line.
	*/
	String processCmdArg(String arg);

	/**
	* Adjusts a set of action inputs for possible config path stripping.
	*
	* <p>For artifacts, this is a no-op.
	*
	* <p>{@link com.google.devtools.build.lib.actions.cache.VirtualActionInput}s materialize on
	* demand, so their behavior may be different. {@link ParamFileActionInput}, particularly, stores
	* a command line in a file. So any paths in that command line changes that input's contents.
	* That's the use case this method handles.
	*/
	List<ActionInput> processInputs(List<ActionInput> inputs);

	/** Instantiates a {@link PathStripper} that doesn't change paths. */
	static PathStripper noop() {
	return new PathStripper() {
	@Override
	public String getExecPathString(ActionInput artifact) {
	return artifact.getExecPathString();
	}

	@Override
	public String processCmdArg(String arg) {
	return arg;
	}

	@Override
	public List<ActionInput> processInputs(List<ActionInput> inputs) {
	return inputs;
	}
	};
	}

	/**
	* Instantiates a {@link PathStripper} for a spawn action.
	*
	* @param spawn the action to support
	* @param pathAgnosticActions mnemonics of actions that "qualify" for path stripping. Just because
	* an action type qualifies doesn't mean its paths are stripped. See {@link
	* #shouldStripPaths}.
	* @param outputRoot root of the output tree ("bazel-out").
	*/
	static PathStripper create(
	Spawn spawn, Collection<String> pathAgnosticActions, PathFragment outputRoot) {
	Preconditions.checkState(outputRoot.isSingleSegment());
	Preconditions.checkState(!outputRoot.getPathString().contains("\\"));
	if (!shouldStripPaths(spawn, pathAgnosticActions, outputRoot)) {
	return noop();
	}
	Pattern stripPathsPattern =
	Pattern.compile("\\Q" + outputRoot.getPathString() + "\\E/[\\w-_]+(/\|$)");

	return new PathStripper() {
	@Override
	public String getExecPathString(ActionInput artifact) {
	if (!isOutputPath(artifact, outputRoot)) {
	return artifact.getExecPathString();
	}
	PathFragment origExecPath = artifact.getExecPath();
	return outputRoot.getRelative(origExecPath.subFragment(2)).toString();
	}

	@Override
	public String processCmdArg(String arg) {
	// Note that we can't just split the input on a simple delimiter like " ". Output paths
	// can be prefixed by all kinds of characters. Examples:
	// params files: "@bazel-out/..."
	// AndroidResourceCompiler: "--resource some_path#bazel-out/x86-opt/some-other-path"
	// busybox.bzl (lots of Android actions): paths prefixed with ",", ":", and more
	//
	// A deeper and probably more sustainable design would be for all actions to store path
	// references as structured inputs (e.g. an Artifact object instead of a path string).
	// Then lazily instantiate their paths in the executor client. That's roughly what
	// processInputs(), below, models for ParamFileActionInputs. But that involves more API
	// modeling and rule logic cleanup. So we take the path of least resistance for now.
	return stripPathsPattern.matcher(arg).replaceAll(outputRoot.getPathString() + "/");
	}

	@Override
	public ImmutableList<ActionInput> processInputs(List<ActionInput> inputs) {
	return inputs.stream()
	.map(
	input ->
	input instanceof ParamFileActionInput
	? ((ParamFileActionInput) input).withAdjustedArgs(this::processCmdArg)
	: input)
	.collect(toImmutableList());
	}
	};
	}

	/**
	* Should this action have its paths stripped for execution?
	*
	* <p>Only true for actions that a) qualify, b) don't have distinct paths that would become
	* duplicates if their config prefixes were removed.
	*/
	static boolean shouldStripPaths(
	Spawn spawn, Collection<String> pathAgnosticActions, PathFragment outputRoot) {
	String actionMnemonic = spawn.getMnemonic();
	if (!pathAgnosticActions.contains(actionMnemonic)) {
	return false;
	}

	// For qualifying action types, check that no inputs or outputs would clash if paths were
	// removed, e.g. "bazel-out/k8-fastbuild/foo" and "bazel-out/host/foo".
	//
	// A more clever algorithm could remap these with custom prefixes - "bazel-out/1/foo" and
	// "bazel-out/2/foo" - if experience shows that would help.
	//
	// Another approach could keep host paths intact (since the "host" path prefix doesn't vary
	// with configurations). While this would help more action instances qualify, it also blocks
	// caching the same action in host and target configurations. This could be mitigated by
	// stripping the host prefix only when the entire action is in the host configuration.
	HashSet<PathFragment> rootRelativePaths = new HashSet<>();
	for (ActionInput input : spawn.getInputFiles().toList()) {
	if (!isOutputPath(input, outputRoot)) {
	continue;
	}
	// For "bazel-out/k8-fastbuild/foo/bar", get "foo/bar".
	if (!rootRelativePaths.add(input.getExecPath().subFragment(2))) {
	return false;
	}
	}
	return true;
	}

	/** Is this a strippable path? */
	static boolean isOutputPath(ActionInput artifact, PathFragment outputRoot) {
	// We can't simply check for DerivedArtifact. Output paths can also appear, for example, in
	// ParamFileActionInput and ActionInputHelper.BasicActionInput.
	return artifact.getExecPath().startsWith(outputRoot);
	}
	}