src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java - bazel - Git at Google

 // Copyright 2016 The Bazel Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package com.google.devtools.build.lib.remote;

 import static java.nio.charset.StandardCharsets.UTF_8;

 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.actions.ActionExecutionContext;
 import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ActionInputHelper;
 import com.google.devtools.build.lib.actions.ExecException;
 import com.google.devtools.build.lib.actions.ExecutionStrategy;
 import com.google.devtools.build.lib.actions.Spawn;
 import com.google.devtools.build.lib.actions.SpawnActionContext;
 import com.google.devtools.build.lib.actions.Spawns;
 import com.google.devtools.build.lib.actions.UserExecException;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.EventHandler;
 import com.google.devtools.build.lib.remote.ContentDigests.ActionKey;
 import com.google.devtools.build.lib.remote.RemoteProtocol.Action;
 import com.google.devtools.build.lib.remote.RemoteProtocol.ActionResult;
 import com.google.devtools.build.lib.remote.RemoteProtocol.Command;
 import com.google.devtools.build.lib.remote.RemoteProtocol.ContentDigest;
 import com.google.devtools.build.lib.remote.RemoteProtocol.ExecuteReply;
 import com.google.devtools.build.lib.remote.RemoteProtocol.ExecuteRequest;
 import com.google.devtools.build.lib.remote.RemoteProtocol.ExecutionStatus;
 import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode;
 import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy;
 import com.google.devtools.build.lib.util.io.FileOutErr;
 import com.google.devtools.build.lib.vfs.Path;
 import io.grpc.StatusRuntimeException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeSet;

 /**
  * Strategy that uses a distributed cache for sharing action input and output files. Optionally this
  * strategy also support offloading the work to a remote worker.
  */
 @ExecutionStrategy(
   name = {"remote"},
   contextType = SpawnActionContext.class
 )
 final class RemoteSpawnStrategy implements SpawnActionContext {
   private final Path execRoot;
   private final StandaloneSpawnStrategy standaloneStrategy;
   private final RemoteActionCache remoteActionCache;
   private final RemoteWorkExecutor remoteWorkExecutor;

   RemoteSpawnStrategy(
       Map<String, String> clientEnv,
       Path execRoot,
       RemoteOptions options,
       boolean verboseFailures,
       RemoteActionCache actionCache,
       RemoteWorkExecutor workExecutor,
       String productName) {
     this.execRoot = execRoot;
     this.standaloneStrategy = new StandaloneSpawnStrategy(execRoot, verboseFailures, productName);
     this.remoteActionCache = actionCache;
     this.remoteWorkExecutor = workExecutor;
   }

   private Action buildAction(
       Collection<? extends ActionInput> outputs, ContentDigest command, ContentDigest inputRoot) {
     Action.Builder action = Action.newBuilder();
     action.setCommandDigest(command);
     action.setInputRootDigest(inputRoot);
     // Somewhat ugly: we rely on the stable order of outputs here for remote action caching.
     for (ActionInput output : outputs) {
       action.addOutputPath(output.getExecPathString());
     }
     // TODO(olaola): Need to set platform as well!
     return action.build();
   }

   private Command buildCommand(List<String> arguments, ImmutableMap<String, String> environment) {
     Command.Builder command = Command.newBuilder();
     command.addAllArgv(arguments);
     // Sorting the environment pairs by variable name.
     TreeSet<String> variables = new TreeSet<>(environment.keySet());
     for (String var : variables) {
       command.addEnvironmentBuilder().setVariable(var).setValue(environment.get(var));
     }
     return command.build();
   }

   /**
    * Fallback: execute the spawn locally. If an ActionKey is provided, try to upload results to
    * remote action cache.
    */
   private void execLocally(
       Spawn spawn, ActionExecutionContext actionExecutionContext, ActionKey actionKey)
       throws ExecException, InterruptedException {
     standaloneStrategy.exec(spawn, actionExecutionContext);
     if (remoteActionCache != null && actionKey != null) {
       ArrayList<Path> outputFiles = new ArrayList<>();
       for (ActionInput output : spawn.getOutputFiles()) {
         outputFiles.add(execRoot.getRelative(output.getExecPathString()));
       }
       try {
         ActionResult.Builder result = ActionResult.newBuilder();
         remoteActionCache.uploadAllResults(execRoot, outputFiles, result);
         remoteActionCache.setCachedActionResult(actionKey, result.build());
         // Handle all cache errors here.
       } catch (IOException e) {
         throw new UserExecException("Unexpected IO error.", e);
       } catch (UnsupportedOperationException e) {
         actionExecutionContext
             .getExecutor()
             .getEventHandler()
             .handle(
                 Event.warn(
                     spawn.getMnemonic() + " unsupported operation for action cache (" + e + ")"));
       }
     }
   }

   private void passRemoteOutErr(ActionResult result, FileOutErr outErr) {
     if (remoteActionCache == null) {
       return;
     }
     try {
       ImmutableList<byte[]> streams =
           remoteActionCache.downloadBlobs(
               ImmutableList.of(result.getStdoutDigest(), result.getStderrDigest()));
       outErr.printOut(new String(streams.get(0), UTF_8));
       outErr.printErr(new String(streams.get(1), UTF_8));
     } catch (CacheNotFoundException e) {
       // Ignoring.
     }
   }

   /** Executes the given {@code spawn}. */
   @Override
   public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext)
       throws ExecException, InterruptedException {
     if (!spawn.isRemotable() || remoteActionCache == null) {
       standaloneStrategy.exec(spawn, actionExecutionContext);
       return;
     }

     ActionKey actionKey = null;
     String mnemonic = spawn.getMnemonic();
     EventHandler eventHandler = actionExecutionContext.getExecutor().getEventHandler();

     try {
       // Temporary hack: the TreeNodeRepository should be created and maintained upstream!
       TreeNodeRepository repository = new TreeNodeRepository(execRoot);
       List<ActionInput> inputs =
           ActionInputHelper.expandArtifacts(
               spawn.getInputFiles(), actionExecutionContext.getArtifactExpander());
       TreeNode inputRoot = repository.buildFromActionInputs(inputs);
       repository.computeMerkleDigests(inputRoot);
       Command command = buildCommand(spawn.getArguments(), spawn.getEnvironment());
       Action action =
           buildAction(
               spawn.getOutputFiles(),
               ContentDigests.computeDigest(command),
               repository.getMerkleDigest(inputRoot));

       // Look up action cache, and reuse the action output if it is found.
       actionKey = ContentDigests.computeActionKey(action);
       ActionResult result = remoteActionCache.getCachedActionResult(actionKey);
       boolean acceptCached = true;
       if (result != null) {
         // We don't cache failed actions, so we know the outputs exist.
         // For now, download all outputs locally; in the future, we can reuse the digests to
         // just update the TreeNodeRepository and continue the build.
         try {
           remoteActionCache.downloadAllResults(result, execRoot);
           return;
         } catch (CacheNotFoundException e) {
           acceptCached = false; // Retry the action remotely and invalidate the results.
         }
       }

       if (remoteWorkExecutor == null) {
         execLocally(spawn, actionExecutionContext, actionKey);
         return;
       }

       // Upload the command and all the inputs into the remote cache.
       remoteActionCache.uploadBlob(command.toByteArray());
       // TODO(olaola): this should use the ActionInputFileCache for SHA1 digests!
       remoteActionCache.uploadTree(repository, execRoot, inputRoot);
       // TODO(olaola): set BuildInfo and input total bytes as well.
       ExecuteRequest.Builder request =
           ExecuteRequest.newBuilder()
               .setAction(action)
               .setAcceptCached(acceptCached)
               .setTotalInputFileCount(inputs.size())
               .setTimeoutMillis(1000 * Spawns.getTimeoutSeconds(spawn, 120));
       // TODO(olaola): set sensible local and remote timouts.
       ExecuteReply reply = remoteWorkExecutor.executeRemotely(request.build());
       ExecutionStatus status = reply.getStatus();
       result = reply.getResult();
       // We do not want to pass on the remote stdout and strerr if we are going to retry the
       // action.
       if (status.getSucceeded()) {
         passRemoteOutErr(result, actionExecutionContext.getFileOutErr());
         remoteActionCache.downloadAllResults(result, execRoot);
         return;
       }
       if (status.getError() == ExecutionStatus.ErrorCode.EXEC_FAILED) {
         passRemoteOutErr(result, actionExecutionContext.getFileOutErr());
         throw new UserExecException(status.getErrorDetail());
       }
       // For now, we retry locally on all other remote errors.
       // TODO(olaola): add remote retries on cache miss errors.
       execLocally(spawn, actionExecutionContext, actionKey);
     } catch (IOException e) {
       throw new UserExecException("Unexpected IO error.", e);
     } catch (InterruptedException e) {
       eventHandler.handle(Event.warn(mnemonic + " remote work interrupted (" + e + ")"));
       Thread.currentThread().interrupt();
       throw e;
     } catch (StatusRuntimeException e) {
       eventHandler.handle(Event.warn(mnemonic + " remote work failed (" + e + ")"));
       execLocally(spawn, actionExecutionContext, actionKey);
     } catch (CacheNotFoundException e) {
       eventHandler.handle(Event.warn(mnemonic + " remote work results cache miss (" + e + ")"));
       execLocally(spawn, actionExecutionContext, actionKey);
     } catch (UnsupportedOperationException e) {
       eventHandler.handle(
           Event.warn(mnemonic + " unsupported operation for action cache (" + e + ")"));
     }
   }

   @Override
   public boolean willExecuteRemotely(boolean remotable) {
     // Returning true here just helps to estimate the cost of this computation is zero.
     return remotable;
   }

   @Override
   public boolean shouldPropagateExecException() {
     return false;
   }
 }
	// Copyright 2016 The Bazel Authors. All rights reserved.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package com.google.devtools.build.lib.remote;

	import static java.nio.charset.StandardCharsets.UTF_8;

	import com.google.common.collect.ImmutableList;
	import com.google.common.collect.ImmutableMap;
	import com.google.devtools.build.lib.actions.ActionExecutionContext;
	import com.google.devtools.build.lib.actions.ActionInput;
	import com.google.devtools.build.lib.actions.ActionInputHelper;
	import com.google.devtools.build.lib.actions.ExecException;
	import com.google.devtools.build.lib.actions.ExecutionStrategy;
	import com.google.devtools.build.lib.actions.Spawn;
	import com.google.devtools.build.lib.actions.SpawnActionContext;
	import com.google.devtools.build.lib.actions.Spawns;
	import com.google.devtools.build.lib.actions.UserExecException;
	import com.google.devtools.build.lib.events.Event;
	import com.google.devtools.build.lib.events.EventHandler;
	import com.google.devtools.build.lib.remote.ContentDigests.ActionKey;
	import com.google.devtools.build.lib.remote.RemoteProtocol.Action;
	import com.google.devtools.build.lib.remote.RemoteProtocol.ActionResult;
	import com.google.devtools.build.lib.remote.RemoteProtocol.Command;
	import com.google.devtools.build.lib.remote.RemoteProtocol.ContentDigest;
	import com.google.devtools.build.lib.remote.RemoteProtocol.ExecuteReply;
	import com.google.devtools.build.lib.remote.RemoteProtocol.ExecuteRequest;
	import com.google.devtools.build.lib.remote.RemoteProtocol.ExecutionStatus;
	import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode;
	import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy;
	import com.google.devtools.build.lib.util.io.FileOutErr;
	import com.google.devtools.build.lib.vfs.Path;
	import io.grpc.StatusRuntimeException;
	import java.io.IOException;
	import java.util.ArrayList;
	import java.util.Collection;
	import java.util.List;
	import java.util.Map;
	import java.util.TreeSet;

	/**
	* Strategy that uses a distributed cache for sharing action input and output files. Optionally this
	* strategy also support offloading the work to a remote worker.
	*/
	@ExecutionStrategy(
	name = {"remote"},
	contextType = SpawnActionContext.class
	)
	final class RemoteSpawnStrategy implements SpawnActionContext {
	private final Path execRoot;
	private final StandaloneSpawnStrategy standaloneStrategy;
	private final RemoteActionCache remoteActionCache;
	private final RemoteWorkExecutor remoteWorkExecutor;

	RemoteSpawnStrategy(
	Map<String, String> clientEnv,
	Path execRoot,
	RemoteOptions options,
	boolean verboseFailures,
	RemoteActionCache actionCache,
	RemoteWorkExecutor workExecutor,
	String productName) {
	this.execRoot = execRoot;
	this.standaloneStrategy = new StandaloneSpawnStrategy(execRoot, verboseFailures, productName);
	this.remoteActionCache = actionCache;
	this.remoteWorkExecutor = workExecutor;
	}

	private Action buildAction(
	Collection<? extends ActionInput> outputs, ContentDigest command, ContentDigest inputRoot) {
	Action.Builder action = Action.newBuilder();
	action.setCommandDigest(command);
	action.setInputRootDigest(inputRoot);
	// Somewhat ugly: we rely on the stable order of outputs here for remote action caching.
	for (ActionInput output : outputs) {
	action.addOutputPath(output.getExecPathString());
	}
	// TODO(olaola): Need to set platform as well!
	return action.build();
	}

	private Command buildCommand(List<String> arguments, ImmutableMap<String, String> environment) {
	Command.Builder command = Command.newBuilder();
	command.addAllArgv(arguments);
	// Sorting the environment pairs by variable name.
	TreeSet<String> variables = new TreeSet<>(environment.keySet());
	for (String var : variables) {
	command.addEnvironmentBuilder().setVariable(var).setValue(environment.get(var));
	}
	return command.build();
	}

	/**
	* Fallback: execute the spawn locally. If an ActionKey is provided, try to upload results to
	* remote action cache.
	*/
	private void execLocally(
	Spawn spawn, ActionExecutionContext actionExecutionContext, ActionKey actionKey)
	throws ExecException, InterruptedException {
	standaloneStrategy.exec(spawn, actionExecutionContext);
	if (remoteActionCache != null && actionKey != null) {
	ArrayList<Path> outputFiles = new ArrayList<>();
	for (ActionInput output : spawn.getOutputFiles()) {
	outputFiles.add(execRoot.getRelative(output.getExecPathString()));
	}
	try {
	ActionResult.Builder result = ActionResult.newBuilder();
	remoteActionCache.uploadAllResults(execRoot, outputFiles, result);
	remoteActionCache.setCachedActionResult(actionKey, result.build());
	// Handle all cache errors here.
	} catch (IOException e) {
	throw new UserExecException("Unexpected IO error.", e);
	} catch (UnsupportedOperationException e) {
	actionExecutionContext
	.getExecutor()
	.getEventHandler()
	.handle(
	Event.warn(
	spawn.getMnemonic() + " unsupported operation for action cache (" + e + ")"));
	}
	}
	}

	private void passRemoteOutErr(ActionResult result, FileOutErr outErr) {
	if (remoteActionCache == null) {
	return;
	}
	try {
	ImmutableList<byte[]> streams =
	remoteActionCache.downloadBlobs(
	ImmutableList.of(result.getStdoutDigest(), result.getStderrDigest()));
	outErr.printOut(new String(streams.get(0), UTF_8));
	outErr.printErr(new String(streams.get(1), UTF_8));
	} catch (CacheNotFoundException e) {
	// Ignoring.
	}
	}

	/** Executes the given {@code spawn}. */
	@Override
	public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext)
	throws ExecException, InterruptedException {
	if (!spawn.isRemotable() \|\| remoteActionCache == null) {
	standaloneStrategy.exec(spawn, actionExecutionContext);
	return;
	}

	ActionKey actionKey = null;
	String mnemonic = spawn.getMnemonic();
	EventHandler eventHandler = actionExecutionContext.getExecutor().getEventHandler();

	try {
	// Temporary hack: the TreeNodeRepository should be created and maintained upstream!
	TreeNodeRepository repository = new TreeNodeRepository(execRoot);
	List<ActionInput> inputs =
	ActionInputHelper.expandArtifacts(
	spawn.getInputFiles(), actionExecutionContext.getArtifactExpander());
	TreeNode inputRoot = repository.buildFromActionInputs(inputs);
	repository.computeMerkleDigests(inputRoot);
	Command command = buildCommand(spawn.getArguments(), spawn.getEnvironment());
	Action action =
	buildAction(
	spawn.getOutputFiles(),
	ContentDigests.computeDigest(command),
	repository.getMerkleDigest(inputRoot));

	// Look up action cache, and reuse the action output if it is found.
	actionKey = ContentDigests.computeActionKey(action);
	ActionResult result = remoteActionCache.getCachedActionResult(actionKey);
	boolean acceptCached = true;
	if (result != null) {
	// We don't cache failed actions, so we know the outputs exist.
	// For now, download all outputs locally; in the future, we can reuse the digests to
	// just update the TreeNodeRepository and continue the build.
	try {
	remoteActionCache.downloadAllResults(result, execRoot);
	return;
	} catch (CacheNotFoundException e) {
	acceptCached = false; // Retry the action remotely and invalidate the results.
	}
	}

	if (remoteWorkExecutor == null) {
	execLocally(spawn, actionExecutionContext, actionKey);
	return;
	}

	// Upload the command and all the inputs into the remote cache.
	remoteActionCache.uploadBlob(command.toByteArray());
	// TODO(olaola): this should use the ActionInputFileCache for SHA1 digests!
	remoteActionCache.uploadTree(repository, execRoot, inputRoot);
	// TODO(olaola): set BuildInfo and input total bytes as well.
	ExecuteRequest.Builder request =
	ExecuteRequest.newBuilder()
	.setAction(action)
	.setAcceptCached(acceptCached)
	.setTotalInputFileCount(inputs.size())
	.setTimeoutMillis(1000 * Spawns.getTimeoutSeconds(spawn, 120));
	// TODO(olaola): set sensible local and remote timouts.
	ExecuteReply reply = remoteWorkExecutor.executeRemotely(request.build());
	ExecutionStatus status = reply.getStatus();
	result = reply.getResult();
	// We do not want to pass on the remote stdout and strerr if we are going to retry the
	// action.
	if (status.getSucceeded()) {
	passRemoteOutErr(result, actionExecutionContext.getFileOutErr());
	remoteActionCache.downloadAllResults(result, execRoot);
	return;
	}
	if (status.getError() == ExecutionStatus.ErrorCode.EXEC_FAILED) {
	passRemoteOutErr(result, actionExecutionContext.getFileOutErr());
	throw new UserExecException(status.getErrorDetail());
	}
	// For now, we retry locally on all other remote errors.
	// TODO(olaola): add remote retries on cache miss errors.
	execLocally(spawn, actionExecutionContext, actionKey);
	} catch (IOException e) {
	throw new UserExecException("Unexpected IO error.", e);
	} catch (InterruptedException e) {
	eventHandler.handle(Event.warn(mnemonic + " remote work interrupted (" + e + ")"));
	Thread.currentThread().interrupt();
	throw e;
	} catch (StatusRuntimeException e) {
	eventHandler.handle(Event.warn(mnemonic + " remote work failed (" + e + ")"));
	execLocally(spawn, actionExecutionContext, actionKey);
	} catch (CacheNotFoundException e) {
	eventHandler.handle(Event.warn(mnemonic + " remote work results cache miss (" + e + ")"));
	execLocally(spawn, actionExecutionContext, actionKey);
	} catch (UnsupportedOperationException e) {
	eventHandler.handle(
	Event.warn(mnemonic + " unsupported operation for action cache (" + e + ")"));
	}
	}

	@Override
	public boolean willExecuteRemotely(boolean remotable) {
	// Returning true here just helps to estimate the cost of this computation is zero.
	return remotable;
	}

	@Override
	public boolean shouldPropagateExecException() {
	return false;
	}
	}