Update from Google.
--
MOE_MIGRATED_REVID=85702957
diff --git a/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java b/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java
new file mode 100644
index 0000000..0e484f7
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java
@@ -0,0 +1,21 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+/**
+ * Marker interface for actions that must be run unconditionally.
+ */
+public interface AlwaysOutOfDateAction {
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java b/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java
new file mode 100644
index 0000000..84f3aef
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java
@@ -0,0 +1,73 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.devtools.build.lib.actions.Action;
+import com.google.devtools.build.lib.rules.test.TestRunnerAction;
+
+/**
+ * Class implements --check_???_up_to_date execution filter predicate
+ * that prevents certain actions from being executed (thus aborting
+ * the build if action is not up-to-date).
+ */
+public final class CheckUpToDateFilter implements Predicate<Action> {
+
+ /**
+ * Determines an execution filter based on the --check_up_to_date and
+ * --check_tests_up_to_date options. Returns a singleton if possible.
+ */
+ public static Predicate<Action> fromOptions(ExecutionOptions options) {
+ if (!options.testCheckUpToDate && !options.checkUpToDate) {
+ return Predicates.alwaysTrue();
+ }
+ return new CheckUpToDateFilter(options);
+ }
+
+ private final boolean allowBuildActionExecution;
+ private final boolean allowTestActionExecution;
+
+ /**
+ * Creates new execution filter based on --check_up_to_date and
+ * --check_tests_up_to_date options.
+ */
+ private CheckUpToDateFilter(ExecutionOptions options) {
+ // If we want to check whether test is up-to-date, we should disallow
+ // test execution.
+ this.allowTestActionExecution = !options.testCheckUpToDate;
+
+ // Build action execution should be prohibited in two cases - if we are
+ // checking whether build is up-to-date or if we are checking that tests
+ // are up-to-date (and test execution is not allowed).
+ this.allowBuildActionExecution = allowTestActionExecution && !options.checkUpToDate;
+ }
+
+ /**
+ * @return true if actions' execution is allowed, false - otherwise
+ */
+ @Override
+ public boolean apply(Action action) {
+ if (action instanceof AlwaysOutOfDateAction) {
+ // Always allow fileset manifest action to execute because it identifies files included
+ // in the fileset during execution time.
+ return true;
+ } else if (action instanceof TestRunnerAction) {
+ return allowTestActionExecution;
+ } else {
+ return allowBuildActionExecution;
+ }
+ }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/Digest.java b/src/main/java/com/google/devtools/build/lib/exec/Digest.java
new file mode 100644
index 0000000..4262711
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/Digest.java
@@ -0,0 +1,182 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import static java.nio.charset.StandardCharsets.US_ASCII;
+
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.MessageLite;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * A utility class for obtaining MD5 digests.
+ * Digests are represented as 32 characters in lowercase ASCII.
+ */
+public class Digest {
+
+ public static final ByteString EMPTY_DIGEST = fromContent(new byte[]{});
+
+ private Digest() {
+ }
+
+ /**
+ * Get the digest from the given byte array.
+ * @param bytes the byte array.
+ * @return a digest.
+ */
+ public static ByteString fromContent(byte[] bytes) {
+ MessageDigest md = newBuilder();
+ md.update(bytes, 0, bytes.length);
+ return toByteString(BaseEncoding.base16().lowerCase().encode(md.digest()));
+ }
+
+ /**
+ * Get the digest from the given ByteBuffer.
+ * @param buffer the ByteBuffer.
+ * @return a digest.
+ */
+ public static ByteString fromBuffer(ByteBuffer buffer) {
+ MessageDigest md = newBuilder();
+ md.update(buffer);
+ return toByteString(BaseEncoding.base16().lowerCase().encode(md.digest()));
+ }
+
+ /**
+ * Gets the digest of the given proto.
+ *
+ * @param proto a protocol buffer.
+ * @return the digest.
+ */
+ public static ByteString fromProto(MessageLite proto) {
+ MD5OutputStream md5Stream = new MD5OutputStream();
+ try {
+ proto.writeTo(md5Stream);
+ } catch (IOException e) {
+ throw new IllegalStateException("Unexpected IOException: ", e);
+ }
+ return toByteString(md5Stream.getDigest());
+ }
+
+ /**
+ * Gets the digest and size of a given VirtualActionInput.
+ *
+ * @param input the VirtualActionInput.
+ * @return the digest and size.
+ */
+ public static Pair<ByteString, Long> fromVirtualActionInput(VirtualActionInput input)
+ throws IOException {
+ CountingMD5OutputStream md5Stream = new CountingMD5OutputStream();
+ input.writeTo(md5Stream);
+ ByteString digest = toByteString(md5Stream.getDigest());
+ return Pair.of(digest, md5Stream.getSize());
+ }
+
+ /**
+ * A Sink that does an online MD5 calculation, which avoids forcing us to keep the entire
+ * proto message in memory.
+ */
+ private static class MD5OutputStream extends OutputStream {
+ private final MessageDigest md = newBuilder();
+
+ @Override
+ public void write(int b) {
+ md.update((byte) b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) {
+ md.update(b, off, len);
+ }
+
+ public String getDigest() {
+ return BaseEncoding.base16().lowerCase().encode(md.digest());
+ }
+ }
+
+ private static final class CountingMD5OutputStream extends MD5OutputStream {
+ private long size;
+
+ @Override
+ public void write(int b) {
+ super.write(b);
+ size++;
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) {
+ super.write(b, off, len);
+ size += len;
+ }
+
+ public long getSize() {
+ return size;
+ }
+ }
+
+ /**
+ * @param digest the digest to check.
+ * @return true iff digest is a syntactically legal digest. It must be 32
+ * characters of hex with lowercase letters.
+ */
+ public static boolean isDigest(ByteString digest) {
+ if (digest == null || digest.size() != 32) {
+ return false;
+ }
+
+ for (byte b : digest) {
+ char c = (char) b;
+ if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) {
+ continue;
+ }
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * @param digest the digest.
+ * @return true iff the digest is that of an empty file.
+ */
+ public static boolean isEmpty(ByteString digest) {
+ return digest.equals(EMPTY_DIGEST);
+ }
+
+ /**
+ * @return a new MD5 digest builder.
+ */
+ public static MessageDigest newBuilder() {
+ try {
+ return MessageDigest.getInstance("md5");
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("MD5 not available");
+ }
+ }
+
+ /**
+ * Convert a String digest into a ByteString using ascii.
+ * @param digest the digest in ascii.
+ * @return the digest as a ByteString.
+ */
+ public static ByteString toByteString(String digest) {
+ return ByteString.copyFrom(digest.getBytes(US_ASCII));
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
new file mode 100644
index 0000000..58e360b
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
@@ -0,0 +1,195 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.packages.TestTimeout;
+import com.google.devtools.build.lib.rules.test.TestStrategy;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat;
+import com.google.devtools.build.lib.rules.test.TestStrategy.TestSummaryFormat;
+import com.google.devtools.build.lib.util.OptionsUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.Options;
+import com.google.devtools.common.options.OptionsBase;
+
+import java.util.Map;
+
+/**
+ * Options affecting the execution phase of a build.
+ *
+ * These options are interpreted by the BuildTool to choose an Executor to
+ * be used for the build.
+ *
+ * Note: from the user's point of view, the characteristic function of this
+ * set of options is indistinguishable from that of the BuildRequestOptions:
+ * they are all per-request. The difference is only apparent in the
+ * implementation: these options are used only by the lib.exec machinery, which
+ * affects how C++ and Java compilation occur. (The BuildRequestOptions
+ * contain a mixture of "semantic" options affecting the choice of targets to
+ * build, and "non-semantic" options affecting the lib.actions machinery.)
+ * Ideally, the user would be unaware of the difference. For now, the usage
+ * strings are identical modulo "part 1", "part 2".
+ */
+public class ExecutionOptions extends OptionsBase {
+
+ public static final ExecutionOptions DEFAULTS = Options.getDefaults(ExecutionOptions.class);
+
+ @Option(name = "verbose_failures",
+ defaultValue = "false",
+ category = "verbosity",
+ help = "If a command fails, print out the full command line.")
+ public boolean verboseFailures;
+
+ @Option(name = "subcommands",
+ abbrev = 's',
+ defaultValue = "false",
+ category = "verbosity",
+ help = "Display the subcommands executed during a build.")
+ public boolean showSubcommands;
+
+ @Option(name = "check_up_to_date",
+ defaultValue = "false",
+ category = "what",
+ help = "Don't perform the build, just check if it is up-to-date. If all targets are "
+ + "up-to-date, the build completes successfully. If any step needs to be executed "
+ + "an error is reported and the build fails.")
+ public boolean checkUpToDate;
+
+ @Option(name = "check_tests_up_to_date",
+ defaultValue = "false",
+ category = "testing",
+ implicitRequirements = { "--check_up_to_date" },
+ help = "Don't run tests, just check if they are up-to-date. If all tests results are "
+ + "up-to-date, the testing completes successfully. If any test needs to be built or "
+ + "executed, an error is reported and the testing fails. This option implies "
+ + "--check_up_to_date behavior."
+ )
+ public boolean testCheckUpToDate;
+
+ @Option(name = "test_strategy",
+ defaultValue = "",
+ category = "testing",
+ help = "Specifies which strategy to use when running tests.")
+ public String testStrategy;
+
+ @Option(name = "test_keep_going",
+ defaultValue = "true",
+ category = "testing",
+ help = "When disabled, any non-passing test will cause the entire build to stop. By default "
+ + "all tests are run, even if some do not pass.")
+ public boolean testKeepGoing;
+
+ @Option(name = "runs_per_test_detects_flakes",
+ defaultValue = "false",
+ category = "testing",
+ help = "If true, any shard in which at least one run/attempt passes and at least one "
+ + "run/attempt fails gets a FLAKY status.")
+ public boolean runsPerTestDetectsFlakes;
+
+ @Option(name = "flaky_test_attempts",
+ defaultValue = "default",
+ category = "testing",
+ converter = TestStrategy.TestAttemptsConverter.class,
+ help = "Each test will be retried up to the specified number of times in case of any test "
+ + "failure. Tests that required more than one attempt to pass would be marked as "
+ + "'FLAKY' in the test summary. If this option is set, it should specify an int N or the "
+ + "string 'default'. If it's an int, then all tests will be run up to N times. If it is "
+ + "not specified or its value is 'default', then only a single test attempt will be made "
+ + "for regular tests and three for tests marked explicitly as flaky by their rule "
+ + "(flaky=1 attribute).")
+ public int testAttempts;
+
+ @Option(name = "test_tmpdir",
+ defaultValue = "null",
+ category = "testing",
+ converter = OptionsUtils.PathFragmentConverter.class,
+ help = "Specifies the base temporary directory for 'blaze test' to use.")
+ public PathFragment testTmpDir;
+
+ @Option(name = "test_output",
+ defaultValue = "summary",
+ category = "testing",
+ converter = TestStrategy.TestOutputFormat.Converter.class,
+ help = "Specifies desired output mode. Valid values are 'summary' to "
+ + "output only test status summary, 'errors' to also print test logs "
+ + "for failed tests, 'all' to print logs for all tests and 'streamed' "
+ + "to output logs for all tests in real time (this will force tests "
+ + "to be executed locally one at a time regardless of --test_strategy "
+ + "value).")
+ public TestOutputFormat testOutput;
+
+ @Option(name = "test_summary",
+ defaultValue = "short",
+ category = "testing",
+ converter = TestStrategy.TestSummaryFormat.Converter.class,
+ help = "Specifies the desired format ot the test summary. Valid values "
+ + "are 'short' to print information only about tests executed, "
+ + "'terse', to print information only about unsuccessful tests,"
+ + "'detailed' to print detailed information about failed test cases, "
+ + "and 'none' to omit the summary.")
+ public TestSummaryFormat testSummary;
+
+ @Option(name = "test_timeout",
+ defaultValue = "-1",
+ category = "testing",
+ converter = TestTimeout.TestTimeoutConverter.class,
+ help = "Override the default test timeout values for test timeouts (in secs). If a single "
+ + "positive integer value is specified it will override all categories. If 4 comma-"
+ + "separated integers are specified, they will override the timeouts for short, "
+ + "moderate, long and eternal (in that order). In either form, a value of -1 tells blaze "
+ + "to use its default timeouts for that category.")
+ public Map<TestTimeout, Integer> testTimeout;
+
+
+ @Option(name = "resource_autosense",
+ defaultValue = "false",
+ category = "strategy",
+ help = "Periodically (every 3 seconds) poll system CPU load and available memory "
+ + "and allow execution of build commands if system has sufficient idle CPU and "
+ + "free RAM resources. By default this option is disabled, and Blaze will rely on "
+ + "approximation algorithms based on the total amount of available memory and number "
+ + "of CPU cores.")
+ public boolean useResourceAutoSense;
+
+ @Option(name = "ram_utilization_factor",
+ defaultValue = "67",
+ category = "strategy",
+ help = "Specify what percentage of the system's RAM Blaze should try to use for its "
+ + "subprocesses. "
+ + "This option affects how many processes Blaze will try to run in parallel. "
+ + "If you run several Blaze builds in parallel, using a lower value for "
+ + "this option may avoid thrashing and thus improve overall throughput. "
+ + "Using a value higher than the default is NOT recommended. "
+ + "Note that Blaze's estimates are very coarse, so the actual RAM usage may be much "
+ + "higher or much lower than specified. "
+ + "Note also that this option does not affect the amount of memory that the Blaze "
+ + "server itself will use. "
+ + "Also, this option has no effect if --resource_autosense is enabled."
+ )
+ public int ramUtilizationPercentage;
+
+ @Option(name = "local_resources",
+ defaultValue = "null",
+ category = "strategy",
+ help = "Explicitly set amount of local resources available to Blaze. "
+ + "By default, Blaze will query system configuration to estimate amount of RAM (in MB) "
+ + "and number of CPU cores available for the locally executed build actions. It would also "
+ + "assume default I/O capabilities of the local workstation (1.0). This options allows to "
+ + "explicitly set all 3 values. Note, that if this option is used, Blaze will ignore "
+ + "both --ram_utilization_factor and --resource_autosense values.",
+ converter = ResourceSet.ResourceSetConverter.class
+ )
+ public ResourceSet availableResources;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java
new file mode 100644
index 0000000..5dc9914
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java
@@ -0,0 +1,73 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.exec;
+
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.EnvironmentalExecException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
+import com.google.devtools.build.lib.analysis.actions.FileWriteActionContext;
+import com.google.devtools.build.lib.events.EventHandler;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.Path;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * A strategy for executing an {@link AbstractFileWriteAction}.
+ */
+@ExecutionStrategy(name = { "local" }, contextType = FileWriteActionContext.class)
+public final class FileWriteStrategy implements FileWriteActionContext {
+
+ public static final Class<FileWriteStrategy> TYPE = FileWriteStrategy.class;
+
+ public FileWriteStrategy() {
+ }
+
+ @Override
+ public void exec(Executor executor, AbstractFileWriteAction action, FileOutErr outErr,
+ ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException {
+ EventHandler reporter = executor == null ? null : executor.getEventHandler();
+ try {
+ Path outputPath = Iterables.getOnlyElement(action.getOutputs()).getPath();
+ try (OutputStream out = new BufferedOutputStream(outputPath.getOutputStream())) {
+ action.newDeterministicWriter(reporter, executor).writeOutputFile(out);
+ }
+ if (action.makeExecutable()) {
+ outputPath.setExecutable(true);
+ }
+ } catch (IOException e) {
+ throw new EnvironmentalExecException("failed to create file '"
+ + Iterables.getOnlyElement(action.getOutputs()).prettyPrint()
+ + "' due to I/O error: " + e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public ResourceSet estimateResourceConsumption(AbstractFileWriteAction action) {
+ return action.estimateResourceConsumptionLocal();
+ }
+
+ @Override
+ public String strategyLocality(AbstractFileWriteAction action) {
+ return "local";
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/OutputService.java b/src/main/java/com/google/devtools/build/lib/exec/OutputService.java
new file mode 100644
index 0000000..88d9b94
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/OutputService.java
@@ -0,0 +1,122 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.exec;
+
+import com.google.devtools.build.lib.actions.BuildFailedException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.vfs.BatchStat;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+
+/**
+ * An OutputService retains control over the Blaze output tree, and provides a higher level of
+ * abstraction compared to the VFS layer.
+ *
+ * <p>Higher-level facilities include batch statting, cleaning the output tree, creating symlink
+ * trees, and out-of-band insertion of metadata into the tree.
+ */
+public interface OutputService {
+
+ /**
+ * @return the name of filesystem, akin to what you might see in /proc/mounts
+ */
+ String getFilesSystemName();
+
+ /**
+ * @return true if the output service uses FUSE
+ */
+ boolean usesFuse();
+
+ /**
+ * @return a human-readable, one word name for the service
+ */
+ String getName();
+
+ /**
+ * Start the build.
+ *
+ * @throws BuildFailedException if build preparation failed
+ * @throws InterruptedException
+ */
+ void startBuild() throws BuildFailedException, AbruptExitException, InterruptedException;
+
+ /**
+ * Finish the build.
+ *
+ * @param buildSuccessful iff build was successful
+ * @throws BuildFailedException on failure
+ */
+ void finalizeBuild(boolean buildSuccessful) throws BuildFailedException, AbruptExitException;
+
+ /**
+ * Stages the given tool from the package path, possibly copying it to local disk.
+ *
+ * @param tool target representing the tool to stage
+ * @return a Path pointing to the staged target
+ */
+ Path stageTool(Target tool) throws IOException;
+
+ /**
+ * @return the name of the workspace this output service controls.
+ */
+ String getWorkspace();
+
+ /**
+ * @return the BatchStat instance or null.
+ */
+ BatchStat getBatchStatter();
+
+ /**
+ * @return true iff createSymlinkTree() is available.
+ */
+ boolean canCreateSymlinkTree();
+
+ /**
+ * Creates the symlink tree
+ *
+ * @param inputPath the input manifest
+ * @param outputPath the output manifest
+ * @param filesetTree is true iff we're constructing a Fileset
+ * @param symlinkTreeRoot the symlink tree root, relative to the execRoot
+ * @throws ExecException on failure
+ * @throws InterruptedException
+ */
+ void createSymlinkTree(Path inputPath, Path outputPath, boolean filesetTree,
+ PathFragment symlinkTreeRoot) throws ExecException, InterruptedException;
+
+ /**
+ * Cleans the entire output tree.
+ *
+ * @throws ExecException on failure
+ * @throws InterruptedException
+ */
+ void clean() throws ExecException, InterruptedException;
+
+ /**
+ * @param file the File
+ * @return true iff the file actually lives on a remote server
+ */
+ boolean isRemoteFile(Path file);
+
+ /**
+ * @param path a fully-resolved path
+ * @return true iff path is under this output service's control
+ */
+ boolean resolvedPathUnderTree(Path path);
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java
new file mode 100644
index 0000000..8ec1e51
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java
@@ -0,0 +1,143 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import static java.nio.charset.StandardCharsets.US_ASCII;
+
+import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.Maps;
+import com.google.common.io.BaseEncoding;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.DigestOfDirectoryException;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.protobuf.ByteString;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
+
+/**
+ * An in-memory cache to ensure we do I/O for source files only once during a single build.
+ *
+ * <p>Simply maintains a two-way cached mapping from digest <--> filename that may be populated
+ * only once.
+ */
+@ThreadSafe
+public class SingleBuildFileCache implements ActionInputFileCache {
+
+ private final String cwd;
+ private final FileSystem fs;
+
+ public SingleBuildFileCache(String cwd, FileSystem fs) {
+ this.fs = Preconditions.checkNotNull(fs);
+ this.cwd = Preconditions.checkNotNull(cwd);
+ }
+
+ // If we can't get the digest, we store the exception. This avoids extra file IO for files
+ // that are allowed to be missing, as we first check a likely non-existent content file
+ // first. Further we won't need to unwrap the exception in getDigest().
+ private final LoadingCache<ActionInput, Pair<ByteString, IOException>> pathToDigest =
+ CacheBuilder.newBuilder()
+ // We default to 10 disk read threads, but we don't expect them all to edit the map
+ // simultaneously.
+ .concurrencyLevel(8)
+ // Even small-ish builds, as of 11/21/2011 typically have over 10k artifacts, so it's
+ // unlikely that this default will adversely affect memory in most cases.
+ .initialCapacity(10000)
+ .build(new CacheLoader<ActionInput, Pair<ByteString, IOException>>() {
+ @Override
+ public Pair<ByteString, IOException> load(ActionInput input) {
+ Path path = null;
+ try {
+ path = fs.getPath(fullPath(input));
+ BaseEncoding hex = BaseEncoding.base16().lowerCase();
+ ByteString digest = ByteString.copyFrom(
+ hex.encode(path.getMD5Digest())
+ .getBytes(US_ASCII));
+ pathToBytes.put(input, path.getFileSize());
+ // Inject reverse mapping. Doing this unconditionally in getDigest() showed up
+ // as a hotspot in CPU profiling.
+ digestToPath.put(digest, input);
+ return Pair.of(digest, null);
+ } catch (IOException e) {
+ if (path != null && path.isDirectory()) {
+ pathToBytes.put(input, 0L);
+ return Pair.<ByteString, IOException>of(null, new DigestOfDirectoryException(
+ "Input is a directory: " + input.getExecPathString()));
+ }
+
+ // Put value into size map to avoid trying to read file again later.
+ pathToBytes.put(input, 0L);
+ return Pair.of(null, e);
+ }
+ }
+ });
+
+ private final Map<ByteString, ActionInput> digestToPath = Maps.newConcurrentMap();
+
+ private final Map<ActionInput, Long> pathToBytes = Maps.newConcurrentMap();
+
+ @Nullable
+ @Override
+ public File getFileFromDigest(ByteString digest) {
+ ActionInput relPath = digestToPath.get(digest);
+ return relPath == null ? null : new File(fullPath(relPath));
+ }
+
+ @Override
+ public long getSizeInBytes(ActionInput input) throws IOException {
+ // TODO(bazel-team): this only works if pathToDigest has already been called.
+ Long sz = pathToBytes.get(input);
+ if (sz != null) {
+ return sz;
+ }
+ Path path = fs.getPath(fullPath(input));
+ sz = path.getFileSize();
+ pathToBytes.put(input, sz);
+ return sz;
+ }
+
+ @Override
+ public ByteString getDigest(ActionInput input) throws IOException {
+ Pair<ByteString, IOException> result = pathToDigest.getUnchecked(input);
+ if (result.second != null) {
+ throw result.second;
+ }
+ return result.first;
+ }
+
+ @Override
+ public boolean contentsAvailableLocally(ByteString digest) {
+ return digestToPath.containsKey(digest);
+ }
+
+ /**
+ * Creates a File object that refers to fileName, if fileName is an absolute path. Otherwise,
+ * returns a File object that refers to the fileName appended to the (absolute) current working
+ * directory.
+ */
+ private String fullPath(ActionInput input) {
+ String relPath = input.getExecPathString();
+ return relPath.startsWith("/") ? relPath : new File(cwd, relPath).getPath();
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java b/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java
new file mode 100644
index 0000000..40fed77
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java
@@ -0,0 +1,37 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.exec;
+
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.analysis.SourceManifestAction;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+/**
+ * A context for {@link SourceManifestAction} that uses the runtime to determine
+ * the workspace suffix.
+ */
+@ExecutionStrategy(contextType = SourceManifestAction.Context.class)
+public class SourceManifestActionContextImpl implements SourceManifestAction.Context {
+ private final PathFragment runfilesPrefix;
+
+ public SourceManifestActionContextImpl(PathFragment runfilesPrefix) {
+ this.runfilesPrefix = runfilesPrefix;
+ }
+
+ @Override
+ public PathFragment getRunfilesPrefix() {
+ return runfilesPrefix;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java
new file mode 100644
index 0000000..6127cee
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java
@@ -0,0 +1,137 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.BaseSpawn;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.util.CommandBuilder;
+import com.google.devtools.build.lib.util.OsUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.util.List;
+
+/**
+ * Helper class responsible for the symlink tree creation.
+ * Used to generate runfiles and fileset symlink farms.
+ */
+public final class SymlinkTreeHelper {
+
+ private static final String BUILD_RUNFILES = "build-runfiles" + OsUtils.executableExtension();
+
+ /**
+ * These actions run faster overall when serialized, because most of their
+ * cost is in the ext2 block allocator, and there's less seeking required if
+ * their directory creations get non-interleaved allocations. So we give them
+ * a huge resource cost.
+ */
+ public static final ResourceSet RESOURCE_SET = new ResourceSet(1000, 0.5, 0.75);
+
+ private final PathFragment inputManifest;
+ private final PathFragment symlinkTreeRoot;
+ private final boolean filesetTree;
+
+ /**
+ * Creates SymlinkTreeHelper instance. Can be used independently of
+ * SymlinkTreeAction.
+ *
+ * @param inputManifest exec path to the input runfiles manifest
+ * @param symlinkTreeRoot exec path to the symlink tree location
+ * @param filesetTree true if this is fileset symlink tree,
+ * false if this is a runfiles symlink tree.
+ */
+ public SymlinkTreeHelper(PathFragment inputManifest, PathFragment symlinkTreeRoot,
+ boolean filesetTree) {
+ this.inputManifest = inputManifest;
+ this.symlinkTreeRoot = symlinkTreeRoot;
+ this.filesetTree = filesetTree;
+ }
+
+ public PathFragment getSymlinkTreeRoot() { return symlinkTreeRoot; }
+
+ /**
+ * Creates a symlink tree using a CommandBuilder. This means that the symlink
+ * tree will always be present on the developer's workstation. Useful when
+ * running commands locally.
+ *
+ * Warning: this method REALLY executes the command on the box Blaze was
+ * run on, without any kind of synchronization, locking, or anything else.
+ *
+ * @param config the configuration that is used for creating the symlink tree.
+ * @throws CommandException
+ */
+ public void createSymlinksUsingCommand(Path execRoot,
+ BuildConfiguration config, BinTools binTools) throws CommandException {
+ List<String> argv = getSpawnArgumentList(execRoot, binTools);
+
+ CommandBuilder builder = new CommandBuilder();
+ builder.addArgs(argv);
+ builder.setWorkingDir(execRoot);
+ builder.build().execute();
+ }
+
+ /**
+ * Creates symlink tree using appropriate method. At this time tree
+ * always created using build-runfiles helper application.
+ *
+ * Note: method may try to acquire resources - meaning that it would
+ * block for undetermined period of time. If it is interrupted during
+ * that wait, ExecException will be thrown but interrupted bit will be
+ * preserved.
+ *
+ * @param action action instance that requested symlink tree creation
+ * @param actionExecutionContext Services that are in the scope of the action.
+ */
+ public void createSymlinks(AbstractAction action, ActionExecutionContext actionExecutionContext,
+ BinTools binTools) throws ExecException, InterruptedException {
+ List<String> args = getSpawnArgumentList(
+ actionExecutionContext.getExecutor().getExecRoot(), binTools);
+ try {
+ ResourceManager.instance().acquireResources(action, RESOURCE_SET);
+ actionExecutionContext.getExecutor().getSpawnActionContext(action.getMnemonic()).exec(
+ new BaseSpawn.Local(args, ImmutableMap.<String, String>of(), action),
+ actionExecutionContext);
+ } finally {
+ ResourceManager.instance().releaseResources(action, RESOURCE_SET);
+ }
+ }
+
+ /**
+ * Returns the complete argument list build-runfiles has to be called with.
+ */
+ private List<String> getSpawnArgumentList(Path execRoot, BinTools binTools) {
+ List<String> args = Lists.newArrayList(
+ execRoot.getRelative(binTools.getExecPath(BUILD_RUNFILES))
+ .getPathString());
+
+ if (filesetTree) {
+ args.add("--allow_relative");
+ args.add("--use_metadata");
+ }
+
+ args.add(inputManifest.getPathString());
+ args.add(symlinkTreeRoot.getPathString());
+
+ return args;
+ }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
new file mode 100644
index 0000000..d9470e4
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java
@@ -0,0 +1,60 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.analysis.SymlinkTreeAction;
+import com.google.devtools.build.lib.analysis.SymlinkTreeActionContext;
+import com.google.devtools.build.lib.analysis.config.BinTools;
+
+/**
+ * Implements SymlinkTreeAction by using the output service or by running an embedded script to
+ * create the symlink tree.
+ */
+@ExecutionStrategy(contextType = SymlinkTreeActionContext.class)
+public final class SymlinkTreeStrategy implements SymlinkTreeActionContext {
+ private final OutputService outputService;
+ private final BinTools binTools;
+
+ public SymlinkTreeStrategy(OutputService outputService, BinTools binTools) {
+ this.outputService = outputService;
+ this.binTools = binTools;
+ }
+
+ @Override
+ public void createSymlinks(SymlinkTreeAction action,
+ ActionExecutionContext actionExecutionContext)
+ throws ActionExecutionException, InterruptedException {
+ Executor executor = actionExecutionContext.getExecutor();
+ try {
+ SymlinkTreeHelper helper = new SymlinkTreeHelper(
+ action.getInputManifest().getExecPath(),
+ action.getOutputManifest().getExecPath().getParentDirectory(), action.isFilesetTree());
+ if (outputService != null && outputService.canCreateSymlinkTree()) {
+ outputService.createSymlinkTree(action.getInputManifest().getPath(),
+ action.getOutputManifest().getPath(),
+ action.isFilesetTree(), helper.getSymlinkTreeRoot());
+ } else {
+ helper.createSymlinks(action, actionExecutionContext, binTools);
+ }
+ } catch (ExecException e) {
+ throw e.toActionExecutionException(
+ action.getProgressMessage(), executor.getVerboseFailures(), action);
+ }
+ }
+}
